{"cells":[{"attachments":{},"cell_type":"markdown","metadata":{"id":"K8rPS5-aTUys"},"source":["# FineTuning BERT for Multi-Class Classification with custom datasets\n","\n","### [Link to Dataset](https://www.kaggle.com/datasets/savasy/ttc4900)"]},{"cell_type":"code","execution_count":5,"metadata":{},"outputs":[],"source":["# !pip install transformers datasets"]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":25729,"status":"ok","timestamp":1625398158764,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"bixw-FrJ-6La","outputId":"850fe9c3-dc21-4036-bcde-b1b0b5c43034"},"outputs":[],"source":["import torch, os\n","import pandas as pd\n","from transformers import pipeline, BertForSequenceClassification, BertTokenizerFast\n","from torch.utils.data import Dataset"]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"elapsed":28,"status":"ok","timestamp":1625398170258,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"x9RjPCc76GjE","outputId":"2956b1e5-7d27-47b9-f70e-dd32944408bc"},"outputs":[{"data":{"text/plain":["'cuda'"]},"execution_count":7,"metadata":{},"output_type":"execute_result"}],"source":["from torch import cuda\n","device = 'cuda' if cuda.is_available() else 'cpu'\n","device"]},{"cell_type":"code","execution_count":8,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"executionInfo":{"elapsed":397,"status":"ok","timestamp":1625398175947,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"XGvn9cWEtXcE","outputId":"8ddd76d1-8416-481f-bad5-52a6d4bb0672"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>4657</th>\n","      <td>teknoloji</td>\n","      <td>acıların kedisi sam çatık kaşlı kedi sam in i...</td>\n","    </tr>\n","    <tr>\n","      <th>3539</th>\n","      <td>spor</td>\n","      <td>g saray a git santos van_persie den forma ala...</td>\n","    </tr>\n","    <tr>\n","      <th>907</th>\n","      <td>dunya</td>\n","      <td>endonezya da çatışmalar 14 ölü endonezya da i...</td>\n","    </tr>\n","    <tr>\n","      <th>4353</th>\n","      <td>teknoloji</td>\n","      <td>emniyetten polis logolu virüs uyarısı telefon...</td>\n","    </tr>\n","    <tr>\n","      <th>3745</th>\n","      <td>spor</td>\n","      <td>beni türk yapın cristian_baroni yıldırım dan ...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["        category                                               text\n","4657  teknoloji    acıların kedisi sam çatık kaşlı kedi sam in i...\n","3539       spor    g saray a git santos van_persie den forma ala...\n","907       dunya    endonezya da çatışmalar 14 ölü endonezya da i...\n","4353  teknoloji    emniyetten polis logolu virüs uyarısı telefon...\n","3745       spor    beni türk yapın cristian_baroni yıldırım dan ..."]},"execution_count":8,"metadata":{},"output_type":"execute_result"}],"source":["ROOT_DIR = '../input/' # If you have downloaded the data in local Machine\n","# ROOT_DIR = '/kaggle/input/ttc4900/'  # If you are using the data directly within Kaggle Kernel\n","\n","df_org= pd.read_csv(ROOT_DIR + \"7allV03.csv\")\n","\n","df_org = df_org.sample(frac=1.0, random_state=42)\n","\n","df_org.head()"]},{"cell_type":"code","execution_count":9,"metadata":{},"outputs":[{"data":{"text/plain":["['teknoloji', 'spor', 'dunya', 'kultur', 'ekonomi', 'saglik', 'siyaset']"]},"execution_count":9,"metadata":{},"output_type":"execute_result"}],"source":["labels = df_org['category'].unique().tolist()\n","labels = [s.strip() for s in labels ]\n","labels"]},{"attachments":{},"cell_type":"markdown","metadata":{},"source":["\n","## Why we need id2label and labe2ids in NLP Projects\n","\n","In NLP tasks, especially those involving classification problems, id2label and label2id dictionaries are used to map class labels (categories) to integer IDs and vice versa. These mappings are essential for various stages of the NLP pipeline, such as data preprocessing, model training, and evaluation.\n","\n","Data preprocessing: In order to feed text data into an NLP model, the text must first be tokenized and then converted into numerical values. Similarly, class labels must also be transformed into numerical representations. The label2id dictionary helps in converting the original class labels into integer IDs.\n","\n","Model training: NLP models usually output probability distributions over classes as their predictions. During training, the model's predictions are compared against the ground truth labels (which have been converted to integer IDs) to compute the loss and optimize the model parameters.\n","\n","Model evaluation and interpretation: Once the model has been trained, its predictions (in the form of integer IDs) need to be mapped back to their original class labels to make the results interpretable. The id2label dictionary is used to perform this conversion.\n","\n","\n","For `BertForSequenceClassification` model as well I need these exact mapping of id2labels and labels2id in dictionary form.\n","\n","### Hence, fefore you start training your model, create a map of the expected ids to their labels with id2label and label2id:"]},{"cell_type":"code","execution_count":10,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["teknoloji\n","spor\n","dunya\n","kultur\n","ekonomi\n","saglik\n","siyaset\n"]}],"source":["for key, value in enumerate(labels):\n","    print(value)"]},{"cell_type":"code","execution_count":11,"metadata":{"executionInfo":{"elapsed":4,"status":"ok","timestamp":1625398177852,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"5jyOH3YGvyYW"},"outputs":[],"source":["NUM_LABELS= len(labels)\n","\n","id2label={id:label for id,label in enumerate(labels)}\n","\n","label2id={label:id for id,label in enumerate(labels)}"]},{"cell_type":"code","execution_count":12,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1625398179284,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"MQ6iJXbFn86y","outputId":"51bcfc45-6c5e-4f88-a971-ac34e7b6d2bd"},"outputs":[{"data":{"text/plain":["{'teknoloji': 0,\n"," 'spor': 1,\n"," 'dunya': 2,\n"," 'kultur': 3,\n"," 'ekonomi': 4,\n"," 'saglik': 5,\n"," 'siyaset': 6}"]},"execution_count":12,"metadata":{},"output_type":"execute_result"}],"source":["label2id"]},{"cell_type":"code","execution_count":13,"metadata":{"executionInfo":{"elapsed":395,"status":"ok","timestamp":1625398182914,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"EJYWAXVovqOQ"},"outputs":[{"data":{"text/plain":["{0: 'teknoloji',\n"," 1: 'spor',\n"," 2: 'dunya',\n"," 3: 'kultur',\n"," 4: 'ekonomi',\n"," 5: 'saglik',\n"," 6: 'siyaset'}"]},"execution_count":13,"metadata":{},"output_type":"execute_result"}],"source":["id2label"]},{"cell_type":"code","execution_count":14,"metadata":{},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>4657</th>\n","      <td>teknoloji</td>\n","      <td>acıların kedisi sam çatık kaşlı kedi sam in i...</td>\n","    </tr>\n","    <tr>\n","      <th>3539</th>\n","      <td>spor</td>\n","      <td>g saray a git santos van_persie den forma ala...</td>\n","    </tr>\n","    <tr>\n","      <th>907</th>\n","      <td>dunya</td>\n","      <td>endonezya da çatışmalar 14 ölü endonezya da i...</td>\n","    </tr>\n","    <tr>\n","      <th>4353</th>\n","      <td>teknoloji</td>\n","      <td>emniyetten polis logolu virüs uyarısı telefon...</td>\n","    </tr>\n","    <tr>\n","      <th>3745</th>\n","      <td>spor</td>\n","      <td>beni türk yapın cristian_baroni yıldırım dan ...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["        category                                               text\n","4657  teknoloji    acıların kedisi sam çatık kaşlı kedi sam in i...\n","3539       spor    g saray a git santos van_persie den forma ala...\n","907       dunya    endonezya da çatışmalar 14 ölü endonezya da i...\n","4353  teknoloji    emniyetten polis logolu virüs uyarısı telefon...\n","3745       spor    beni türk yapın cristian_baroni yıldırım dan ..."]},"execution_count":14,"metadata":{},"output_type":"execute_result"}],"source":["df_org.head()"]},{"attachments":{},"cell_type":"markdown","metadata":{},"source":["### Create a new column to represent the categories in numerical form\n","\n","I need a 'label' column heading with numeric value else while running the epochs with `trainer.train()` I will get below error\n","\n","```\n","BertForSequenceClassification ValueError: The model did not return \n","\n","```\n","\n","\n","### In below I am doing it manually, but I could have also done it with pd.factorize() as below\n","\n","Pandas factorize method is used for encoding categorical variables as integers. It assigns a unique integer value to each distinct category in a given Series or DataFrame, effectively transforming non-numeric data into numeric values."]},{"cell_type":"code","execution_count":15,"metadata":{},"outputs":[],"source":["# df_org['labels_num'] = pd.factorize(df_org.category)[0]\n","# df_org.head()"]},{"cell_type":"code","execution_count":16,"metadata":{"executionInfo":{"elapsed":8,"status":"ok","timestamp":1625398182915,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"HbdFN3VZoawV"},"outputs":[],"source":["df_org[\"labels\"]=df_org.category.map(lambda x: label2id[x.strip()])"]},{"cell_type":"code","execution_count":17,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":204},"executionInfo":{"elapsed":6,"status":"ok","timestamp":1625398183337,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"KGAcx7nqwJgm","outputId":"5836de30-de03-480c-8db3-2ab403ac142f"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>category</th>\n","      <th>text</th>\n","      <th>labels</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>4657</th>\n","      <td>teknoloji</td>\n","      <td>acıların kedisi sam çatık kaşlı kedi sam in i...</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3539</th>\n","      <td>spor</td>\n","      <td>g saray a git santos van_persie den forma ala...</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>907</th>\n","      <td>dunya</td>\n","      <td>endonezya da çatışmalar 14 ölü endonezya da i...</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>4353</th>\n","      <td>teknoloji</td>\n","      <td>emniyetten polis logolu virüs uyarısı telefon...</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3745</th>\n","      <td>spor</td>\n","      <td>beni türk yapın cristian_baroni yıldırım dan ...</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["        category                                               text  labels\n","4657  teknoloji    acıların kedisi sam çatık kaşlı kedi sam in i...       0\n","3539       spor    g saray a git santos van_persie den forma ala...       1\n","907       dunya    endonezya da çatışmalar 14 ölü endonezya da i...       2\n","4353  teknoloji    emniyetten polis logolu virüs uyarısı telefon...       0\n","3745       spor    beni türk yapın cristian_baroni yıldırım dan ...       1"]},"execution_count":17,"metadata":{},"output_type":"execute_result"}],"source":["df_org.head()"]},{"cell_type":"code","execution_count":18,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":483},"executionInfo":{"elapsed":712,"status":"ok","timestamp":1625398187147,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"zeNYOXNBGe1B","outputId":"68de8dcd-b392-4915-e5c4-5d21f5004958"},"outputs":[{"data":{"text/plain":["<Axes: ylabel='count'>"]},"execution_count":18,"metadata":{},"output_type":"execute_result"},{"data":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAo8AAAJ8CAYAAACflkvMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAACDGklEQVR4nO3dd3hUVeI+8PfOJJM26T0Qegi9o9I7CgIiiooNbLu2RVRA/VlZ19X1KyuoCFgQUawIWCiCSCiRnkILIQkJ6b1O2rT7+wPNGmmTZGbOnZn38zw8yGTmzpuYZN45955zJFmWZRARERERWUAlOgAREREROQ6WRyIiIiKyGMsjEREREVmM5ZGIiIiILMbySEREREQWY3kkIiIiIouxPBIRERGRxVgeiYiIiMhiLI9EREREZDGWRyIiIiKyGMsjEREREVmM5ZGIiIiILMbySEREREQWY3kkIiIiIouxPBIRERGRxVgeiYiIiMhiLI9EREREZDGWRyIiIiKyGMsjEREREVmM5ZGIiIiILMbySEREREQWY3kkIiIiIouxPBIRERGRxVgeiYiIiMhiLI9EREREZDGWRyIiIiKyGMsjEREREVmM5ZGIiIiILMbySEREREQWY3kkIiIiIouxPBIRERGRxVgeiYiIiMhiLI9EREREZDGWRyIiIiKyGMsjEREREVmM5ZGIiIiILMbySEREREQWY3kkIiIiIouxPBIRERGRxVgeiYiIiMhiLI9EREREZDGWRyIiIiKyGMsjEREREVmM5ZGIiIiILMbySEREREQWY3kkIiIiIouxPBIRERGRxVgeiYiIiMhiLI9ERKR4sizDaDSKjkFEYHkkIqJW2LBhA/r27QsvLy8EBwdj4sSJqK2tBQDMmzcPM2fOxJIlSxAaGgo/Pz88/PDD0Ov1TY9vbGzE/PnzERYWBk9PT4wcORJHjhxp+nhcXBwkScK2bdswePBgeHh4YP/+/Xb/PInoYm6iAxARkWMpKCjAnDlz8Oabb+Lmm29GTU0N9u3bB1mWm+6za9cueHp6Ii4uDllZWbjvvvsQHByM1157DQCwePFifPfdd/j000/RsWNHvPnmm7j++uuRnp6OoKCgpuM8++yzeOutt9ClSxcEBgba/XMlootJ8p9/2omIiK4iISEBgwcPRlZWFjp27HjRx+fNm4cff/wROTk58Pb2BgCsWrUKixYtQlVVFerr6xEYGIi1a9fizjvvBAAYDAZ06tQJCxYswKJFixAXF4dx48Zh8+bNuOmmm+z6+RHRlfG0NRERtUj//v0xYcIE9O3bF7Nnz8aHH36IioqKi+7zR3EEgGHDhkGn0yEnJwcZGRkwGAwYMWJE08fd3d1xzTXXICUlpdlxhgwZYttPhohajOWRiIhaRK1WY+fOndi2bRt69eqFd999F7GxscjMzLT6c/n4+Fj9mETUNiyPRETUYpIkYcSIEViyZAkSExOh0WiwadOmpo8nJyejvr6+6d8HDx6EVqtFdHQ0unbtCo1Gg/j4+KaPGwwGHDlyBL169bLr50FELccJM0RE1CKHDh3Crl27MHnyZISFheHQoUMoKSlBz549m+6j1+vxwAMP4IUXXkBWVhZefvllPP7441CpVPDx8cEjjzyCRYsWISgoCB06dMCbb76Juro6PPDAAwI/MyKyBMsjERG1iJ+fH/bu3Ytly5ahuroaHTt2xNKlSzFlypSm+0yYMAExMTEYPXo0GhsbMWfOHLzyyitNH3/jjTdgNptxzz33oKamBkOGDMHPP//MGdVEDoCzrYmIyKrmzZuHyspKbN68WXQUIrIBXvNIRERERBZjeSQiIiIii/G0NRERERFZjCOPRERERGQxlkciIiIishjLIxERERFZjOWRiIiIiCzG8khEREREFmN5JCIiIiKLsTwSERERkcVYHomIiIjIYiyPRERERGQxN9EBiIgUpVEHNFYDjTUX/jRUAXodYDa1/FiSCvDQAh5+v//x/d8fSbJ+diIiO2B5JCLnVVcOVOcBVXkX/q7OA+rKgIY/lcOmolh9oTjKrSiJLSWpAI329yL5p1Lp+ft/ewcDvlGAfzvArx3g3x7wCbF9LiIiC3BvayJyTA1VzUvhH/9dlfv7bfmAoU50Sutx8wR8Iy8USb92vxfLKMCv/f9KpneQ6JRE5AJYHolI2eorgZIzQPFpoPj3v0vOALUlopMpj1cgENoTCOsBhPUCQn//2ydYdDIiciIsj0SkDI2630tiSvOyWJMvOpnj8wn9X5H8c7H0ChCdjIgcEMsjEdlfQzWQdwzIPXrh76JTQFUOAP46sivfKCCsJ9BuENB+6IU/PPVNRFfB8khENiXLMkoqMhCWuR/IPXKhMJaeBWSz6Gh0KUFd/lcko68BwvsAKrXoVESkICyPRGRVBrMBKWUpSChKwLHiY0gsToSH5I5dKQmio1FrePhdKJIdhwEdhgPtBgPunqJTEZFALI9E1GZnys9gf95+HMw/iOOlx1FvrL/oPjsqzIiszBWQjqxK7QFEDQS6jAG6TbpQJlXcb4LIlbA8ElGL6fQ6HCg4gH25+xCfF4/i+uKrPuZNjy6YcibO9uHIvryDga7jgZjJQLeJvGaSyAWwPBKRRdIq0rAvbx/25e5DUkkSjGZjix4/J6Av/l/iFhulI0WQVEDUoAtFMmbShRFK7qRD5HRYHonokuoMdThYcBD78vZhf95+FNYWtul4PX074Zvje62UjhyCTxjQbcKFItl1ApcGInISLI9E1KTWUItfzv+CbVnbcLjgMAxmg9WOrZbU+C23GN76WqsdkxyIpAY6DAP63Az0nsXT20QOjOWRyMXpTXrszd2LrZlbsTd3LxpNjTZ7ro8Qjmszj9js+OQgVO5A13FAn1uBHjcCHlrRiYioBdxEByAi+zOZTThUeAhbz23Fr9m/osZQY5fnTfQLxbV2eSZSNLMBSNtx4Y+7N9D9BqDvrRdmb7tpRKcjoqtgeSRyIcklydh6bit+zvoZZQ1ldn/+JFXLJtmQCzDUAac2XvjjGQD0mnFhRLLTKC4BRKRQPG1N5OTOVZ3DTxk/YVvmNuTqxK6z6OuuRfzZFEjchpCuxjcS6H0z0O92IGqA6DRE9Ccsj0ROyGg2Ylf2Lnyd+jWOFCrrGsONdV6IKUoVHYMcSdQgYOiDQJ9buLsNkQKwPBI5kaLaImxI24CNZzdatHC3CC95x2L2qZ2iY5Aj8goEBtwFDH3gwh7cRCQEyyOREzhYcBBfn/kacTlxMMrKvq5wRmAfvJawVXQMcmjShV1thj54YbINr40ksiuWRyIHVaOvwffp3+Obs98gsypTdByLdfCOxJZTh0THIGfh3wEYPBcYNBfQhopOQ+QSWB6JHExqeSq+PPMltmZuRb2xXnScVokrqUewrkR0DHImag3Qc8aF0ciOw0SnIXJqLI9EDuJA/gF8eOJDxU2AaY1lbh0wIW2/6BjkrCIHAKOeBnpO597aRDbAdR6JFEyWZcTlxOGjEx/heOlx0XGsJlnrjwmiQ5DzKkgCvrkHCO0JjHrqwixtlVp0KiKnwZFHIgUyy2bsyNqBD098iLMVZ0XHsboBfl3xWfJu0THIVQR2BkY+CfSfwx1siKyA5ZFIQYxmI3469xM+PvExsqqzRMexGY1KgwNZOdDYcB9toov4tQOGz78wwcbdS3QaIofF8kikAI2mRmxK24RPTn6C/Np80XHs4jNjEAbkJImOQa7IJxQY9tiFyTUevqLTEDkclkcigeoMdfj27Lf49NSnKKl3rdnHT/v2xrzj20THIFfmGQBc+3fg2ocB7yDRaYgcBsujCxs7diwGDBiAZcuWiY7icoxmI749+y1WJa9CeUO56DhCjA/sheUJ20XHIAI0vsDIJ4Bhj/N0NpEFONuayM52Ze/CsmPLnPqaRksk1xWIjkB0gb4G+PVfwNFPgHHPX5hYw11riC6LPx1EdnKi5ATmbpuLBbsXuHxxBICyxgrkBHcSHYPof6rzgO8fBVaPBjK4GgDR5bA8uoja2lrce++90Gq1iIyMxNKlSy+6jyRJ2Lx5c7PbAgICsHbtWgBAVlYWJEnCxo0bMW7cOHh7e6N///44cOBA03P4+flhw4YNzY6xefNm+Pj4oKamBgDwzDPPoHv37vD29kaXLl3w4osvwmAwWP+TVojcmlws2rMId229CwnFCaLjKEpiWBfREYguVnQC+Gwm8PktQNEp0WmIFIfl0UUsWrQIe/bswffff48dO3YgLi4OCQmtKzLPP/88Fi5ciKSkJHTv3h1z5syB0WiEj48P7rjjDnzyySfN7v/JJ5/g1ltvha/vhVmNvr6+WLt2LU6fPo3ly5fjww8/xNtvv93mz1Fpqhqr8H9H/g8zNs/A9qztkMHLi/8qyYvXl5GCpf8CrBoJfP8YUM3LLIj+wGseXYBOp8PHH3+Mzz//HBMmXNjX49NPP0X79u1bdbyFCxfixhtvBAAsWbIEvXv3Rnp6Onr06IEHH3wQw4cPR0FBASIjI1FcXIytW7fil19+aXr8Cy+80PTfnTp1wsKFC/HVV19h8eLFbfgslcNgMuCLM1/gg+MfoFpfLTqOoiUaKkVHILoy2Qwkfg6c3HhhQs2IJwAPrehUREJx5NEFZGRkQK/X49prr226LSgoCLGxsa06Xr9+/Zr+OzIyEgBQXFwMALjmmmvQu3dvfPrppwCAzz//HB07dsTo0aObHvP1119jxIgRiIiIgFarxQsvvIDs7OxWZVGan7N+xozNM/DW0bdYHC1wrjYPNZ7+omMQXZ2hDtj7JvDuoAsTa8xm0YmIhGF5pCaSJOGvKzdd6lpEd3f3Zo8BAPOffpE++OCDTddJfvLJJ7jvvvua7nfgwAHcddddmDp1Kn766SckJibi+eefh16vt/anY1c5NTl4eOfDWLhnIXJ1uaLjOAyzbEZyVE/RMYgspysCfloArJkMFJ4UnYZICJZHF9C1a1e4u7vj0KFDTbdVVFTg7NnmeyaHhoaioOB/1/WkpaWhrq6uxc9399134/z583jnnXdw+vRpzJ07t+ljv/32Gzp27Ijnn38eQ4YMQUxMDM6fP9+Kz0oZjGYjPj7xMWZ9Pwvx+fGi4zikJL9g0RGIWi73CPDBGGDnS4ChXnQaIrviNY8uQKvV4oEHHsCiRYsQHByMsLAwPP/881D9ZR2z8ePH47333sOwYcNgMpnwzDPPNBtltFRgYCBmzZqFRYsWYfLkyc2urYyJiUF2dja++uorDB06FFu2bMGmTZva/DmKcLzkOJYcWIKzFWevfme6rCSJ+1uTgzIbgfjlwKnNwLT/At0mik5EZBcceXQR//d//4dRo0Zh+vTpmDhxIkaOHInBgwc3u8/SpUsRHR2NUaNG4c4778TChQvh7e3dqud74IEHoNfrcf/99ze7fcaMGXjyySfx+OOPY8CAAfjtt9/w4osvtvrzEkGn1+G1g6/hnm33sDhawQldLkySWnQMotarPH9hWZ8NDwA619pmlFwTtyckm/jss8/w5JNPIj8/HxqNRnQcq9l5fifeOPQGiuuLRUdxKt80aNGz4LToGERt5xkATFoCDJoL/H6tN5Gz4cgjWVVdXR0yMjLwxhtv4O9//7vTFMfC2kL8Y9c/8FTcUyyONpAY3Lplo4gUp6ES+PEJ4JOpQEmq6DRENsHySFb15ptvokePHoiIiMBzzz0nOk6bmcwmrDu1DjdtvglxuXGi4zitJA0vvyYnk/3bhQXGf30NMPK6XnIuPG1NdBk51Tl4dv+zOF5yXHQUpxfpFYodp4+JjkFkGyGxwC0fApH9RSchsgqWR6JL2JS2CW8cfgN1xpYvVUSt80u5EeFV+aJjENmGWgOMfwEYPp/XQpLDY3kk+pOqxiosObAEO8/vFB3F5byl6YzrU/eIjkFkW51HAzevBvyiRCchajVe80j0u0MFhzDrh1ksjoIkeXO/YHIBmXuBlcMvrA1J5KBYHsnlGUwGLD26FA/teAjFdZxJLUqSWSc6ApF91FcA384FNj8GNPL7nhwPT1uTSztXeQ7P7HsGZ8rPiI7i8twkN/yWWwgvPa8zJRcS2Bm45SOg/RDRSYgsxpFHcllfnvkSt/90O4ujQhhlI05G9hIdg8i+KjKBNdcDcf8BzCbRaYgswvJILqesvgyP7XoM/z70bzSYGkTHoT9JCggVHYHI/sxGIO7fFxYWr8gSnYboqlgeyaUcKzqGW3+8FXtz94qOQpeQpDKLjkAkTs5BYNVo4MwW0UmIrojlkVzGulPr8ODPD6K0vlR0FLqM5Lo8yOAaeOTCGquAr+4CflkCmPlmipSJE2bI6dUZ6vDyby9je9Z20VHIAt/XatClOF10DCLxuowDbvkY8AkWnYSoGY48klPLqsrCnVvuZHF0IEkhnURHIFKGc7uBD8YAedy6k5SF5ZGc1u7s3ZizZQ4yqjJER6EWSPT0EB2BSDmqcmD+Yg5+PHZOdBKiJiyP5HRkWcaq5FV4YvcT0Bm4AK+jSdKXi45ApBiypMJS7yfwj29T8PL3J2E08TpIEo/XPJJTqTPU4YX4F7jFoIPbV6RDQB1LJNGe6EcwN21U07+HdQnG+3cNQqCPRmAqcnUceSSnkafLw73b7mVxdAJJUT1FRyASrqDd9c2KIwAcOFeGGSv240xhtaBURCyP5CSOFh7FnJ/mILUiVXQUsoIk3wDREYiEagyKxbScOy/5sZzyesx6/zdsP1lg51REF7A8ksPbnrkdf9v5N1Q0VoiOQlaSKNeLjkAkjOzhj/vrF6BM737Z+9TpTXh0fQI+ic+0YzKiC1geyaF9dvozLN67GAazQXQUsqLTuhwYVJd/4SRyVrKkwjL/RYiv8L/qfc0ysOTH03h9awo4fYHsieWRHJIsy3jryFt488ibkMFfms6mwdSIFF73SC4ovv1DWJ7dpUWPWb33HJ76JhkGzsQmO2F5JIdjMBvw7L5n8enpT0VHIRtKCowSHYHIrgqjJuGe9NGteuymxDzcv/YIahuNVk5FdDGWR3IotYZaPPrLo9iauVV0FLKxJJ61JheiD4zBjLy7Icut39t9X1opbv/gAEpqGq2YjOhiLI/kMErrSzFv+zwcLDgoOgrZQVJ9oegIRHYhe/jhwcYnUdzY9ndMJ/OqMWtlPDJLa62QjOjSWB7JIWRWZeLurXfjTPkZ0VHITkoaypEX1EF0DCKbkiHhHf/F2FseYLVj5pTX45aVvyExmytQkG2wPJLiJZck495t9yJPlyc6CtlZYlhX0RGIbOpA9EN4u4UTZCxRXqvHnR8ewq9niqx+bCKWR1K0uJw4PPjzg6hsrBQdhQRI8vYWHYHIZoqjJuCu9DE2O369wYSH1h3D10eybfYc5JpYHkmxfs76GU/ufhINpgbRUUiQJGOV6AhENqEP6IoZefe0aYKMJUxmGc98dwJruZg4WRHLIynSz1k/49m9z8Ioc9kJV5auy0Oth6/oGERWJWu0+Jv+KRQ2auz2nK/8eJoFkqyG5ZEUh8WR/mCSTUjmYuHkRGRIWBG4GHHlgXZ/bhZIshaWR1IUFkf6q2T/ENERiKzmcPQDeOt8N2HPzwJJ1sDySIrB4kiXkiRx33JyDqVRY3FH+jjRMVggqc1YHkkRWBzpco7X5sIs8VcVOTaDfxdMz59n8wkylnrlx9P49Lcs0THIQfE3MgnH4khXojPUIi08VnQMolaTNT542PgUChrsN0HGEi//cIoFklqF5ZGEYnEkSySFRIuOQNQqMiSsDlyEXWVBoqNcEgsktQbLIwnD4kiWStK4iY5A1CoJ0fPwxvnuomNcEQsktRTLIwmx8/xOFkeyWGJjiegIRC1WFjkGt6VPEB3DIi//cArrDmSJjkEOguWR7O5QwSE8s/cZFkeyWF5dEUp9w0XHILKYwb8TZhTMg0l2nJfZl384hR+S80XHIAfgON/V5BRSy1OxYPcCGMxcfoVaJilS2af+iP4gu/vgMdNC5DV4iI7SIrIMLPwmGb+ll4qOQgrH8kh2k6/LxyO/PAKdQSc6CjmgRB8/0RGILPJx8ELsKFXmBJmr0ZvM+Ptnx3A6v1p0FFIwlkeyi6rGKjz8y8Moqee1a9Q6SeY60RGIrioxei7+leXYS0vVNBox75PDyK3gzxxdGssj2VyDsQGP73ocmVXc0YBaL0WXjUY3T9ExiC6rPGIkbk2fJDqGVRTXNGLumsOorNOLjkIKxPJINmUym7B472IklSSJjkIOzmA24FRUL9ExiC7J4NcRM4vud6gJMleTUVKLBz49igaDSXQUUhjn+S4nRXrt0GvYnbNbdAxyEokBnHFNyiO7e2O+/DSy651vZPzY+Qr848tEmMyy6CikICyPZDOrklfh27Pfio5BTiRJbRYdgegia0OexraSENExbGbn6SK8+P1J0TFIQVgeySY2pW3CiqQVomOQkzleXyA6AlEzydH3YElmT9ExbO6LQ9l4d1ea6BikECyPZHV7c/finwf+KToGOaHyxkpkhXQRHYMIAFAZMRy3ZtwgOobdLN15Ft8cyREdgxSA5ZGsKr0iHYv2LOLuMWQzSWEsjySe0S8aM4sehMEsiY5iV/9v0wkcPFcmOgYJxvJIVlOtr8YTu59AnZFrg5HtJHk61q4d5HxkNy88iYXIcsIJMldjNMt4/IsE5FfWi45CArE8klWYZTOe3fsssmuyRUchJ5dkqBAdgVzc56FP4cfiUNExhCnV6fHw58e4hI8LY3kkq3gv8T3sy9snOga5gHO6PFR5BYiOQS7qRPRdeDGzt+gYwh3PrcILmzkD21WxPFKb7Tq/Cx+d+Eh0DHIRMmQkRzn/7FZSnqrw63BrxhTRMRRjw7FcrDuQJToGCcDySG1yrvIcno9/HjK4gCzZT5JvkOgI5GKMvu0wq+QhNJr5svlnr/50Goczy0XHIDvjTwG1mk6vwxO7n0CtoVZ0FHIxSWgQHYFciOzmiYWqRcio8xIdRXEMJhmPrk9AYRV/Jl0JyyO1iizLeG7fc8iqzhIdhVzQSV0OjCo30THIRXwV9iQ2F4WJjqFYpbpGPPz5MTQaOYHGVbA8UqusTF6JuNw40THIRdWbGpAa0UN0DHIBp6Pn4LlzfUXHULyknEq8tPmU6BhkJyyP1GJxOXFYlbxKdAxycUnB7UVHICdXHX4Nbs64UXQMh/H10Rx8fvC86BhkByyP1CJZVVl4bt9znCBDwiW6u9bOHmRfJm0Ubin9OyfItNA/fzyNY+c5gcbZ8aeCLGYwGbB472LoDDrRUYiQ1FAkOgI5KVntgcXqRUir5QSZltKbzJj/ZRKqGwyio5ANsTySxd5NfBcp5SmiYxABAIrqS1EYwFPXZH3fhD+J74rCRcdwWHmV9XhhExcQd2Ysj2SRQwWHsPbUWtExiJpJDO8mOgI5mdTo2/DMuX6iYzi8H5LzsSkxV3QMshGWR7qqqsYqPL+fC4GT8iT5aEVHICdSEzYEM89NFx3Daby0+RRyyutExyAbYHmkq1pyYAmK6nh9GSlPorFadARyEiZtJGaXP4x6k1p0FKdR02jEk18nwWTmwIOzYXmkK9qUtgk7z+8UHYPoktJ0uajT+IiOQQ5OVmvwrNsinNF5i47idI6er8CK3emiY5CVsTzSZeVU5+CNw2+IjkF0WUbZiBNRPUXHIAe3MWIBvi2MEB3Dab2zKw2J2RWiY5AVsTzSJRnNRjy771nUGXm9Cilbkj+3jaPWS4u+FU9nDBAdw6kZzTIWfJ2E2kaj6ChkJSyPdEmrklfheOlx0TGIripJxRckah1d6CDMPDdTdAyXcL6sDq/8wO0LnQXLI10ksTgRH534SHQMIosk1+ZCBneboZYx+YRjdsWjqDXxZdBevj2Wi60nCkTHICvgTw01o9Pr8Ny+52CSTaKjEFmkxqBDRnh30THIgchqDV7ULEYKJ8jY3XMbT6Cgql50DGojlkdq5q2jbyFPlyc6BlGLJIZ0EB2BHMj3EfPxRUGk6BguqaregOc2nhAdg9qI5ZGaJBQlYGPaRtExiFosyUMjOgI5iIzoW7AgY5DoGC4tLrUEPybni45BbcDySAAAg8mAJQeWcBcZckhJjaWiI5ADqA0dgBmZM0XHIABLfjyNqnqD6BjUSiyPBAD4+OTHOFd1TnQMolbJritAuU+I6BikYGbvUNxR+RhqjdxBRglKdY14Y9sZ0TGolVgeCVlVWfjw+IeiYxC1SRIXC6fLkFXueMXzGZyo4W5ESvLVkWwcySoXHYNageWR8OrBV6E360XHIGqTJK2f6AikUFuiHse6/CjRMegvZBn4fxtPQG80i45CLcTy6OI2p2/G4cLDomMQtVmSzOU/6GKZ7Wfi8fShomPQZaQV67B6T4boGNRCLI8urKKhAkuPLhUdg8gqTtVkw6DmrGv6n7qQ/piWdYvoGHQV7+1OR2ZpregY1AIsjy7sraNvobKxUnQMIqvQm/U4FdVLdAxSCLN3CO6o5gQZR9BoNOP5TVz70ZGwPLqogwUH8UPGD6JjEFlVUmCE6AikALLKDf/0XIzj1VrRUchCv2WU4btjuaJjkIVYHl1Qo6kRrx54VXQMIqtL4iATAdge9RjW5rcXHYNa6LWtKSiv5eRNR8Dy6II+OP4BsmuyRccgsrqk+gLREUiw8+1n4JH0a0XHoFYor9XjtS0pomOQBVgeXUy+Lh+fnvpUdAwimyhrrEBOcEfRMUiQ+pA+mHH+VtExqA02JubieG6l6BitNnbsWCxYsMCmz7F27VoEBAS06DGdOnXCsmXLmv4tSRI2b97c6gwsjy7mncR30GhqFB2DyGaSwrqKjkACmL2CcVfNfFQZ3ERHoTaQZShi9NEeJdCejhw5gr/97W9N/y4oKMCUKVNafTyWRxdyquwUtp7bKjoGkU0lenmJjkB2JktqvOa9GAlVnCDjDA5llmPn6SLRMZxKaGgovL29m/4dEREBDw+PVh+P5dGFLD26FDJk0TGIbCrJUCU6AtnZznaP4eO8aNExyIre2JYCo0nMzjPz5s3Dnj17sHz5ckiSBEmSkJWVBQA4efIkpkyZAq1Wi/DwcNxzzz0oLS297LG2bNkCf39/rF+/vunYM2fOxFtvvYXIyEgEBwfjscceg8FgaHpMRUUF7r33XgQGBsLb2xtTpkxBWlraFTOvXLkSXbt2hUajQWxsLD777LNmH+dpa2qVuJw4HCk8IjoGkc1l1OaixtNfdAyyk5z2N+Jv6deJjkFWllFSiy+P5Ah57uXLl2PYsGF46KGHUFBQgIKCAkRHR6OyshLjx4/HwIEDcfToUWzfvh1FRUW47bbbLnmcL774AnPmzMH69etx1113Nd2+e/duZGRkYPfu3fj000+xdu1arF27tunj8+bNw9GjR/HDDz/gwIEDkGUZU6dObVYw/2zTpk144okn8PTTT+PkyZP4+9//jvvuuw+7d++26tflz1geXYDRbMR/j/1XdAwiuzDLZhyP6iE6BtlBfXBvTD9/6RducnzLfzkLXaPR7s/r7+8PjUYDb29vREREICIiAmq1Gu+99x4GDhyIf//73+jRowcGDhyINWvWYPfu3Th79myzY6xYsQKPPvoofvzxR0ybNq3ZxwIDA/Hee++hR48emDZtGm688Ubs2rULAJCWloYffvgBH330EUaNGoX+/ftj/fr1yMvLu+xI4VtvvYV58+bh0UcfRffu3fHUU09h1qxZeOutt2zy9QFYHl3CxrSNyKzKFB2DyG6S/EJERyAbM3sF4R7dfFQa3EVHIRsp1ekVte91cnIydu/eDa1W2/SnR48Lb1QzMv6Xc8OGDXjyySexc+dOjBkz5qLj9O7dG2r1/xaljYyMRHFxMQAgJSUFbm5uuPba/y03FRwcjNjYWKSkXHoiUUpKCkaMGNHsthEjRlz2/tbAaWlOrtZQi/eT3hcdg8iuEsEVBZyZLKnxH5/FOJrrKzoK2dhH+zJx17UdEeHvKToKdDodpk+fjv/85z8XfSwyMrLpvwcOHIiEhASsWbMGQ4YMgSRJze7r7t78DY8kSTCbxVzf2VoceXRya06uQVlDmegYRHZ1ojYXJonbzTirX9s/gtW5HUTHIDuoN5iwdEeq3Z9Xo9HAZDI1u23QoEE4deoUOnXqhG7dujX74+Pj03S/rl27Yvfu3fj+++/xj3/8o0XP27NnTxiNRhw6dKjptrKyMqSmpqJXr16XfUx8fHyz2+Lj4y97f2tgeXRiRbVF+Oz0Z1e/I5GTqTPW4WxErOgYZAN57abggbThomOQHX2XkIuUgmq7PmenTp1w6NAhZGVlobS0FGazGY899hjKy8sxZ84cHDlyBBkZGfj5559x3333XVQ0u3fvjt27d+O7775r0XqRMTExuOmmm/DQQw9h//79SE5Oxt1334127drhpptuuuRjFi1ahLVr12LlypVIS0vDf//7X2zcuBELFy5sy5fgilgendh7Se+h3lgvOgaREEnBXLrF2TQE9cS07DmiY5CdmWXg31vtu3D4woULoVar0atXL4SGhiI7OxtRUVGIj4+HyWTC5MmT0bdvXyxYsAABAQFQqS6uU7Gxsfj111/x5Zdf4umnn7b4uT/55BMMHjwY06ZNw7BhwyDLMrZu3XrR6e4/zJw5E8uXL8dbb72F3r17Y/Xq1fjkk08wduzY1n76VyXJssyF/5xQankqbvvpNphlx7qOgshapgT2wZsJXBTfWZg9AzAHb+BQpZ/oKCTIp/dfgzHdQ0XHcEiRkZF49dVX8eCDD1rleBx5dFIrklawOJJLS24oFh2BrESWVFjqu5jF0cX9V8C1j46urq4OO3fuRFFREXr37m2147I8OqHU8lTE5cSJjkEkVH59MYr9I69+R1K8ve3/jhU5nUTHIMGSc6uwO5VvClvigw8+wB133IEFCxZg2LBhVjsuT1s7oafinsLO8ztFxyAS7i1NJ1yfuld0DGqDgnbXY1jGXNExSCEGdgjApkdHXP2OZFMceXQyaRVp+OX8L6JjEClCkjdPczqyxqBYTMu5U3QMUpDE7ErsPVsiOobLY3l0MquPr4YMDiYTAUCSuUZ0BGol2cMf99cvQJmeO8hQc+/sShMdweWxPDqRc5XneLqa6E/O6HLQ4O4lOga1kCypsMx/EeIr/EVHIQU6er4C8emlomO4NJZHJ6J+/3O8fG4AAs18sSQCAKPZiBNRtttlgWwjvv1DWJ7dRXQMUrB3f+Xoo0gsj05Cn5uHui++Rc+vDuODVRLeSB+ECJNWdCwi4ZIDwkRHoBYojJqEe9JHi45BCnfwXDkSsytEx3BZLI9OonzNGsBoBADIVdXo8u1hvLvCgP+eHoiOxgCx4YgESlJxvVNHoQ+MwYy8uyHLkugo5ABWxmWIjuCyuFSPEzCWlyN9/ATIDQ2X/Ljk6YmiCX3xbs8cpLrzOhFyLQEaf+xNPQmJE8kUTfbww1z1G9hbHiA6CjkISQJ2Pjka3cJ8RUdxORx5dALl69ZdtjgCgNzQgLAtR/Dq8jKsPtYfA/QRdkxHJFalvgqZYV1Fx6ArkCHhHf/FLI7UIrIMrIw7JzqGS2J5dHAmXS0qvvjSsjsbDAjccQz/b1kBPj7YF8Ma2ts2HJFCJIV0FB2BruBA9EN4mxNkqBV+SM5DfmW96Bguh+XRwVV++y3M1dUte5DJBN/diXhy2Xl8urc3xtd1skk2IqVI8vQUHYEuozhqAu5KHyM6Bjkog0nGmv2ZomO4HJZHBybLMiq+tHDU8dIHgFd8Mh5eno7Pf+mBabpu1gtHpCCJ+jLREegS9AFdMSPvHk6QoTb55mgO6vUm0TFcCsujA6vdvx+G7GyrHEtz5CTuffcMvtzaDbdX9bDKMYmU4nxtASq9g0THoD+RNVr8Tf8UChs1oqOQg6tuMOKH5DzRMVwKy6MDs/haxxZQJ5/BLe+fxNebO2FeeS9InKBKTkCGjOSonqJj0O9kSFgRuBhx5YGio5CT+OzgedERXArLo4My5OVBt2ePzY4vpaRj6urj+Prbdni0pB/U4GklcmyJvgGiI9DvDkc/gLfO8zIZsp6TedVcNNyOWB4dVMXX3wBmOyx+nHEeYz9KwFfrw/BU4QB4yGrbPyeRDSTJl1/OiuynNGos7kgfJzoGOSGOPtoPFwl3QLJej7Rx42Eqs/8kACkiDAkTO+Dd9qehk/R2f36i1vJUe+C3c1lwNxtER3FZBv8uGF31EgoaeJ0jWZ+HmwoHn5uAQB9+f9kaRx4dUPXPO4QURwCQC4sx8POjWPuRJ5ZkDUKg2UtIDqKWajA14gyvexRG1vjgYeNTLI5kM41GM745miM6hktgeXRAbVqex0rMpeXo+eVhfLBKwhtpgxBm9hEdieiqEgOjREdwSTIkrA5chF1lnPFOtrX+UDZ4QtX2WB4dTENqKuoTEkTHaCJXVaPLhsNYscKI/6YMQgdTgOhIRJeV5M6JXyIc63Af3jjfXXQMcgHZ5XWIO1siOobTY3l0MLZYnscaZF0t2m8+jKXv6vDu8YGIMQaLjkR0keT6QtERXE5Z5BjcnjZedAxyIZ8f4MQZW+OEGQdi0umQPnoMzHV1oqNcnbs7ysf1x/t9C3BcUyQ6DVGT7ZVAuwrrLK5PV2bw74SxVS8jr8FDdBRyISoJ2Lt4HNoHeouO4rQ48uhAqjZ/7xjFEQAMBgTtOIoXlhXio4P9MKyhvehERACApPCuoiO4BNndB4+ZFrI4kt2Z5QvXPpLtsDw6kMrvvhMdoeVMJvjtTsCTy85j7d7eGFffSXQicnGJ3hyNsIePgxdiRyknyJAYG47lwmTmiVVbYXl0EI0ZGWhMSREdo/VkGd7xyXhkWTo+39UD03TcXYLESDZWi47g9BKj5+JfWbGiY5ALK6lpxIEMMUvauQKWRwdR9dNPoiNYjebwSdz77hl8ubUbbqvuIToOuZg0XS5qPXxFx3Ba5REjcWv6JNExiPB9Up7oCE6L5dFBVG/ZKjqC1amTz+DWFSfx1fedMK+8NySeYSA7MMkmHOdi4TZh8OuImUX3wyTzpYXE236qEI1Gk+gYTok/4Q6g/vhxGLKd9+Jf1el0TF2djK83tMcjJf2gBtfiI9tK8g8VHcHpyO7emC8/jex6T9FRiAAANQ1G7D7DNR9tgeXRAVRv2SI6gn2kZ2HcRwn4an0YnizoD42sFp2InFQS92W3urUhT2NbSYjoGETN/JicLzqCU+I6jwonm81IHzMWxhLXe/ckRYYjYWIHvNvuFHR8sScr0rr7ID4tFSrZLDqKU0iOvgc3pU0RHYPoIp7uKhx9YRK0Hm6iozgVjjwqXN3hwy5ZHAFALijCwM+OYO1HXnglaxD8zTwdRtahM9QiLYzb5VlDZcRw3Jpxg+gYRJfUYDBjxynuLGVtLI8K50yzrFvLXFqGXl8exker1Xg9fRDCzD6iI5ETSA7tIDqCwzP6RWNm0YMwmHmdMinX90k8dW1tLI8KJuv1qNmxU3QMxZArq9D128NYscKIpWcGoYMpQHQkcmCJGp7GagvZzQtPYiGyOEGGFC4+vRTltbz0yZpYHhVMt28fzNVc0PivZF0tojcdxtJ3dXj3xEDEGINFRyIHlNRYKjqCQ/s89Cn8WMxZ66R8RrOMLcc5+mhNLI8K5jKzrFtJrm9A+E9H8NqycqxKGIB++nDRkciB5NYVolQbJjqGQzoRfRdezOwtOgaRxX7grGurYnlUKHNtLWp2x4mO4RgMBgT9fBQvLCvERwf74drGdqITkYNIiuQWei1VFX4dbs3gzGpyLEfPVyCvsl50DKfB8qhQur17IdfzG71FTCb47U7A08tysHZfb4yr7yQ6ESlcktZfdASHYvRth1klD6HRzJcOciyyDGw7USA6htPgbwCF0u3ZKzqC4zKb4b0/GY8sS8dnu3rgRl030YlIoRLNOtERHIbs5omFqkXIqPMSHYWoVeJSXXPZO1tgeVQgWZah279fdAyn4HH4JOa+ewZfbOuG2dU8RUnNpehy0OjG2cKW+CrsSWwu4jWi5LgOZ5ajTm8UHcMpsDwqUMPJUzCVciaoNbklncHsFafw1fedMLei94VzGOTyDGYDTkX2FB1D8U5Hz8Fz5/qKjkHUJnqTGfvT+NpqDSyPCqTbu0d0BKelOp2OG1cl45sN0Xi4pC/U4OLGri4pMEJ0BEWrDr8GN2fcKDoGkVXs5qlrq+AquQpUu3ef6AjOLz0L49OzMKFje/w2Phwrwk9CL5lEpyIBktw4Cn05Jm0Ubin9OyfIkNPYk1osOoJT4G8EhTFWVKD+xAnRMVyGfD4Xwz45hvXrAvFc3kB4m91FRyI7S67j+m+XIqs9sFi9CGm1nCBDziO/qgGphTWiYzg8lkeFqd2/HzCbRcdwOXJ+IQauO4J1H3vjlayB8DdzEoWrKG+sxPmQLqJjKM434U/iuyIuvE/OZzdHH9uM5VFhuESPWObSMvT68gg+Wq3G6+mDEGbSio5EdpAY2ll0BEVJjb4Nz5zrJzoGkU3sPsPy2FYsjwoim80XRh5JOLmyCl2/PYwVK41YemYQ2hu5mLQzS/LiSPMfasKGYOa56aJjENnMsfMVqGkwiI7h0FgeFaTh+HGYKitFx6A/kWt0iN50GG+/V4t3TwxEjDFYdCSygSRDhegIimDSRuKWskdQb1KLjkJkM0azzCV72ojlUUF0e3nKWqnk+gaE/3QEry0rx8qE/uir52LJzuScLg/VXq49uiyrNXjWbRHOcoIMuQBe99g2LI8KwusdHYDBgOCfj+HF5cX46FA/XNvYTnQisgIZMpKjeomOIdTGiAX4tpBrXpJr4FaFbcPyqBDGigo0nD4tOgZZymiE368JeHpZDtbu64Ox9R1FJ6I2SvQNEh1BmLToW/F0xgDRMYjsprimESkF1aJjOCyWR4WoT0jglnmOyGyG9/4kPLosA5/t6oGptV1FJ6JWSkaj6AhC6EIHYea5maJjENnd0axy0REcFsujQtQnJoqOQG3kcfgk5r2Tii+2x2B2VazoONRCJ2pzYFS51qZbJp9wzK54FLUmvhSQ6zl2nhPlWou/MRSiLoHl0Vm4JaZg9vun8NUPnXFvRW/RcchC9cZ6pEb0EB3DbmS1Bi9qFiNF5y06CpEQR1keW43lUQFkvR4Np06JjkFWpjqVhmmrkvHNhvZ4uLQv1JBER6KrSApynQlQ30fMxxcFkaJjEAmTW1GP4uoG0TEcEsujAjScPg250TWvt3IJaVkY/2EivvoiHAsK+0Mjcw09pUrSuMavxIzoW7AgY5DoGETCcfSxdVzjN6XC8ZS1a5DP52L4J8ewfl0gns0dCG+zu+hI9BdJDc6/9ltt6ADMyJwpOgaRIvC6x9ZheVQATpZxLXJ+IQZ9dgSfrvHBy+cHwd/MrfGUorC+BIUBznvq2uwdijsqH0OtkaPfRABHHluL5VEB6pJYHl2RXFKK3l8cxkcfuOHfGYMQZtKKjkQAksJjREewCVnljlc8n8GJGh/RUYgU43R+FRoMJtExHA7Lo2D6nByYSrjHpiuTKyrR7ZvDWLHSiLfODEJ7o2tvkydaordzlqstUf/Auvwo0TGIFMVgkpGcUyk6hsNheRSMp6zpD3KNDh02Hcbb79XhnRMDEWMIFh3JJSWZakRHsLrM6Jl4PH2I6BhEinQsm6euW4rlUbC6hATREUhh5Pp6RPx0BP9+pwIrEwagjyFcdCSXclaXizqN84w+1oX0x7TMW0THIFKsBF732GIsj4LVJyaJjkAKJev1CP75KF5aVoSPDvfDNY3OO5FDSYyyESejeomOYRVm7xDcUc0JMkRXwhnXLcfyKJBJV4vGtDTRMUjpjEb47UrAwmU5+GR/H4yt7yg6kdNL8g8VHaHNZJUb/um5GMerORGL6Eoq6gzIKNGJjuFQWB4FajyTApjNomOQozCb4bMvCY8uy8Bnv/bElNquohM5rUSVUXSENtse9RjW5rcXHYPIIZzMqxIdwaGwPArEUUdqLY9DJ3DfO6n4YnsMbq2OFR3H6Ryvy4PswNtJnm8/A4+kXys6BpHDSCviyGNLsDwK1JiWLjoCOTi3xBTctuIUvvqhM+6tcI7r9JSgWl+DDAdd77E+pA9mnL9VdAwih5Ja5HyrLNgSy6NAHHkka1GdSsO0VcfxzYZo/L20L9QOPGqmFEkhjndtqdkrGHfVzEeVwU10FCKHksby2CIsjwI1pnPkkawsLRMTPkzEV1+G44mi/tDInGXbWokejrX3uCyp8Zr3YiRUcYIMUUtll9dxp5kWYHkUxFhaClMFlwcg25CzcjFizTGsXxeIZ/IGwNvsWEVICZL1ZaIjtMjOdo/h47xo0TGIHJJZBtKLed2jpVgeBeEpa7IHOb8Qg9cdxadrfPDy+UHwlz1FR3IY52sLUO4TIjqGRXLa34i/pV8nOgaRQ0st5KlrS7E8CsLJMmRPckkpen9xGB+tdsO/MwYhzMRTm5ZIiuohOsJV1Qf3xvTzt4mOQeTwzvK6R4uxPArCkUcSQa6oRLdvDmPFSiPeSh2E9kZ/0ZEULUmr7K+P2SsI9+jmo9LAyxKI2orl0XIsj4JwsgyJJNfo0GHjYbz9Xh3eOTEQXY1BoiMpUpK5TnSEy5IlNf7jsxhHq3xFRyFyCme51qPFWB4FYXkkJZDr6xHx0xG8sbwSKxMHoI8hXHQkRTldmwuDWiM6xiX92v4RrM7tIDoGkdPIq6yHrtHxd5eyB5ZHAQyFhTDXcHiclEPW6xG8/SheWlaEDw/3w9DGKNGRFKHR1IhTUT1Fx7hIXrspeCBtuOgYRE6Hp64tw/IoAK93JMUyGuG/KwGLluXik/29Mabe8RbKtrbkwEjREZppCOqJadlzRMcgckpcLNwyLI8CNGZkiI5AdGVmM3z2JeOxZRn4bHdPTKntKjqRMEluytmtx+wZgLl1T6CCO8gQ2QT3uLYMy6MAhvx80RGILOZx8ATueycVX2yPwS01saLj2F1SXYHoCAAAWVJhqe9iHKr0Ex2FyGnlVdaLjuAQWB4FMBYUio5A1GJuiSm4/b1T+OqHzrinQnnXAdpKaWM5coLFn77f0/5hrMjpJDoGkVMrqGoQHcEhsDwKYChkeSTHpTqVhumrTuCb76Lxt9I+UEM5p3VtJSlM7Gn7/HY3YF7aSKEZiFxBQRVHHi3B8iiAoVAZp8GI2uRsJiZ+mISvvgzHE4X9oZHVohPZTJKXl7DnbgyKxfQcTpAhsoeSmkYYTGbRMRSP5dHOZL0eptIy0TGIrEbOysWIT45h/bpAPJM3EN5m59vtJNFQKeR5ZQ9/3F+/AGV65/uaEimRWQaKqnnq+mpYHu3MUFwMyLLoGERWJ+cXYvC6I/h0jQ9eOj8Q/rKn6EhWk1GbB52nfSeqyJIKy/wXIb5C2VskEjkbXvd4dSyPdmYs4Clrcm5ySSn6fHEEH612w2sZgxBi9hEdqc3MshnHo3rZ9Tnj2z+E5dld7PqcRATkc8b1VbE82pmhsEh0BCK7kCsqEfPNYax834z/Sx2EdibHXmIm0S/Ybs9VGDUJ96SPttvzEdH/cOTx6lge7YyTZcjVyDU16LjxMJa9W4/lJweiqzFIdKRWSZL0dnmexsDumJ57N2TZ+WexEylRAUcer4rl0c64xiO5Krm+HpE/HsEbyyvxfuIA9NaHiY7UIid0OTBJtp1RLnv44aHGBSjhBBkiYfI58nhVLI92xjUeydXJej1Cth/Fy8uL8eHhfhjSGCU6kkVqjXVIi7DdDjsyJLzjvxh7ywNs9hxEdHVc6/HqWB7tjKetiX5nNMJ/VwIWL8vFJ/F9MKZe/C4uV5MY3N5mxz4Q/RDe5gQZIuEKKjnyeDUsj3bG09ZEf2E2w2dvEh5bloHPdvfEDbVid3O5kiSNm02OWxw1AXelj7HJsYmoZcpq9Wg0mkTHUDSWRzuSjUaYKipExyBSLI+DJ3D/O6n44ufumFXTXXSciyQ3llj9mPqArpiRdw8nyBApSElNo+gIisbyaEdmnU50BCKH4JZwGne8dxpf/dgFd1fad33FK8mrK0Kxf6TVjid7+OJv+qdQ2Kix2jGJqO1qGoyiIygay6MdmVgeiVpEdfIsZqw8jm++i8bfSvtAUsDmTEkRMVY5jgwJKwIWIa480CrHIyLrYXm8MpZHO+LII1Ernc3ExA+T8M1XkZhf1B8a2bZL5lxJorevVY5zOPoBvHW+m1WORUTWpWs0iI6gaK0qj+PHj0dlZeVFt1dXV2P8+PFtzeS0zDU1oiMQOTQ5Kwcj1xzD+s+CsDh/ADxl20xguZJkc22bj1EaNRZ3pI+zQhoisgWOPF5Zq8pjXFwc9PqLd1toaGjAvn372hzKWfG0NZF1yHkFGPLpUXy2RouXzg+Ev+xpt+dO0WWjwd2r1Y83+HfB9Px5nCBDpGAsj1fWorftx48fb/rv06dPo/BPC16bTCZs374d7dq1s146J2PWtX3Egoj+Ry4uRZ8vSvFRYADOTh6EtzulolRl258zo9mIk5G9MCT7WIsfK2t88LDxKRQ0cIIMkZLpGlker6RF5XHAgAGQJAmSJF3y9LSXlxfeffddq4VzNmYdT1sT2YJcUYmYrw9jpa8vsiYPwrKu6chTV9vs+ZICwzEku2WPkSFhdeAi7DrvmHt7E7mSmgZe83glLSqPmZmZkGUZXbp0weHDhxEaGtr0MY1Gg7CwMKjV4i5kVzqetiayLbmmBh2/O4xlXl7InzQI78RmIcOt3OrPk6Rq+QLCx6Ln4Y005a1dSUQX0/G09RW1qDx27Hhh+zCz2WyTMM7OXMPySGQPcn09In84jDc0GpSMH4AVvfNxSlNsteMn1+VDhgQJlq0dVBY5BrenT7Da8xORbfGaxytr9VTFtLQ07N69G8XFxReVyZdeeqnNwZwRl+ohsi9Zr0fI9qN4+Rc3VI3tj9X9SnDUI7/Nx63UVyEztAu6lGRc9b4G/06YUTAPJpkroxE5ihpe83hFrSqPH374IR555BGEhIQgIiICkvS/WYOSJLE8Xoa5luWRSAijEf6/HMPiX1WoHdkfawZXYZ9nCy9a/Ivk0M5XLY+yuw8eMy1EXoNHm56LiOyLp62vrFXl8V//+hdee+01PPPMM9bO49RMPG1NJJbZDJ+9ifjHPgkPXdsH669txM/e51p1qERPD9x8lft8HLwQO7I4QYbI0dRwkfAratV5lIqKCsyePdvaWZweT1sTKYQsw/PgCTyw/CzW7+iOm6tbvuVgkv7KE3ESo+fiX1mxrU1IRAJx5PHKWlUeZ8+ejR07dlz1fmPHjsWCBQta8xQALixGLknSJXezcUTmxgbREYjoL9yPncacFSn46qcuuKuyp8WPy6rNR5X3pfelLo8YiVvTJ1krIhHZWYOBE4OvpFWnrbt164YXX3wRBw8eRN++feHu7t7s4/Pnz7dKuL9au3YtFixY4Lhl0sRvRiKlUp04i5tOADNiu+CXMf74KOgErrQJjAwZSZE9MSbjt2a3G/w6YkbhA5wgQ+TATLJlKym4qlaVxw8++ABarRZ79uzBnj17mn1MkiSblUdrMhgMF5Vem+MSR0SKJ6Wew6RUYHLnDtg7PhQrQ0/AKF36ZzfJLxBj/vRv2d0b8+WnkcsJMkQOTWZ5vKJWvTXOzMy87J9z5y5/8fmWLVvg7++P9evXIysrC5IkISkpqenjlZWVkCQJcXFxFz02Li4O9913H6qqqpp2uXnllVcAXCismzdvbnb/gIAArF27FgCanuvrr7/GmDFj4OnpifXr17fmU28TmeWRyGHImdkY9fExfPlZMBblDYCnfPF77US5+aUoa0OexraSEHtFJCIbMZlZHq+k1es8ttQXX3yBhx9+GF988QWmTZuGrKysFj1++PDhWLZsGV566SWkpqYCALRabYuO8eyzz2Lp0qUYOHAgPD09W/RYq2B5JHI4cl4Bhq4rwGfhoTg+sTOWtT+FGlUjAOCULhsGlTvczQYkR9+DJWmWXzNJRMrF7nhlrSqP999//xU/vmbNmmb/XrFiBZ5//nn8+OOPGDNmzGUedWUajQb+/v6QJAkRERGtOsaCBQswa9asVj3WGmRzy7c0IyJlkItK0Hd9CdYEBSJ1Um8s65SKUtTiTGQPRJt8cWvGDaIjEpGVmNker6hV5bGioqLZvw0GA06ePInKykqMHz++2cc2bNiA4uJixMfHY+jQoa1PagVDhgwR+vx8K0Pk+OTyCnT/+jBW+voia/IgnOnriX+kTobBfIXZNUTkUMy85vGKWlUeN23adNFtZrMZjzzyCLp27drs9oEDByIhIQFr1qzBkCFDmnajUakuXG7554tSDYbWLcopSdJFF7de6lg+Pj6tOj4R0V8Z3b3hqZoCdV4j1iMSMnhmgchZSFCLjqBoVltLQqVS4amnnsLbb7/d7PauXbti9+7d+P777/GPf/yj6fbQ0FAAQEFBQdNtf548cykajQYm08W/oENDQ5sdJy0tDXV1da35NGxLxZEJIkdnCuuA/Nv/hd+GvoJTJWHIyz+Gk/IBK/42JSLh+HJ9RVadMJORkQGj8eJV2bt3747du3dj7NixcHNzw7Jly+Dl5YXrrrsOb7zxBjp37ozi4mK88MILVzx+p06doNPpsGvXLvTv3x/e3t7w9vbG+PHj8d5772HYsGEwmUx45pln7L8MjwUkFd/JEDkqY/sYFI56EOdK/WAsMgMwQ+1mQkVBNspyzyH8us4IKQoVHZOIrIGDPVfUqvL41FNPNfu3LMsoKCjAli1bMHfu3Es+JjY2Fr/++ivGjh0LtVqNpUuXYs2aNXjggQcwePBgxMbG4s0338TkyZMv+7zDhw/Hww8/jNtvvx1lZWV4+eWX8corr2Dp0qW47777MGrUKERFRWH58uU4duxYaz4121JxaILI0Ri69EP+tXORWewFc4EM4H+rJvgFV6O25MIb5l0H12DWiMVwz+eLDpGjk/hjfEWS3IqVMMeNG9fs3yqVCqGhoRg/fjzuv/9+uLnZbQUgh5I56xY0nD4tOgYRWUDfYyhyB92N84VukC+zylZY9FlkH/+p6d8enj64uc+TkMtad/02ESmDSuuOqBeuEx1DsVrV8nbv3m3tHK6BI49EitfYbzRy+sxGdoEKyL/yffV1Oc0f21CLuNyvMDboNsj1nEBD5LA48nhFbRoiLCkpaVqwOzY2tmkSDF2GmuWRSKnqr5mK7K7TkFcgX7U0Ahf2tq4oyLjo9sLCdJwOOoyeqsF/PsNNRA5E4nnrK2pVm6mtrcX999+PyMhIjB49GqNHj0ZUVBQeeOABZc5yVgiVh4BdbYjoiupGzMKpO1bigPeNF4qjhXwD69FYq7vkx46f/gVlYaXWikhEdia5c7DnSlr11XnqqaewZ88e/Pjjj6isrERlZSW+//577NmzB08//bS1MzoNVQu3UyQi25AlCTXj7sLx21bjoPsEFBW2fIjQ26fsih//5cDHMEa1NiERiSR5cu7GlbTqq/Pdd99hw4YNGDt2bNNtU6dOhZeXF2677TasXLnSWvmcitqX5ZFIJFmlRs3EecjwuxYVpQag+OKlxSxlNl393PaWYysws/cCyOWcQEPkSFQeXFrvSlpVHuvq6hAeHn7R7WFhYTxtfQUqH5ZHIhHMGg9UT3oQGR79UVVuAErbXuZqSjOvep+GBh325H2NMYGzITdwAg2Ro+DI45W16rT1sGHD8PLLL6OhoaHptvr6eixZsgTDhg2zWjhnw9PWRPZl9vRB+U1P4eiUd5BQ2+tCcbQCjacRVSUFV78jgILCNJxRHeXsTSIHwpHHK2tVtV62bBluuOEGtG/fHv379wcAJCcnw8PDAzt27LBqQGei4mlrIrswawNQNvkRZOg7oa7KCKD1p6cvxTeoHNUtmFyTdGoHwoZ1RGBhkFVzEJFtSJ4sj1fSqvLYt29fpKWlYf369Thz5gwAYM6cObjrrrvg5eVl1YDORM2RRyKbMgWGoXTiw0ivjUJjuQnWLo1/UKuKWvyYHQc/wq3DF0NtwTJARCSWiqetr6hVX53XX38d4eHheOihh5rdvmbNGpSUlOCZZ56xSjhno9L6io5A5JRMoe1RPP5hZFSFQF9iAmDb6wvrq7Nb/iBZxk8JKzGz13xOoCFSOBVHHq+oVdc8rl69Gj169Ljo9t69e2PVqlVtDuWsVFof0RGInIqxXTfkznkD8QOfR0pRIPR2mJQiqcyoKLj6ZJlLaaivxr6CbyHxeioiRZM8OPJ4Ja366hQWFiIyMvKi20NDQ1FQYNlF5K6Ip62JrMPQuQ/yh92HzCIvmAtk2HMrF79gHYrK9K1+fF5+KlKDE9Bd3x+w/LJJIrIjjjxeWavKY3R0NOLj49G5c+dmt8fHxyMqiqviXo7Kl6etidpC32Mo8gbdhaxCd8j5gIj25eFZ0uZjJJ7YjrDhHRFQEND2QERkdVyq58pa9dV56KGHsGDBAhgMBowfPx4AsGvXLixevJg7zFwBl+ohap2GfqOQ2+c2ZBeoLNp32paMjXlWOc7PBz7ArcOegboFs7aJyD64VM+Vtao8Llq0CGVlZXj00Ueh1184fePp6YlnnnkGzz33nFUDOhOetiZqmfqhNyC724wLe04rZJZyZWGGdQ4ky9ia9D5m9JgPuYITaIiUhCOPVybJstzqt706nQ4pKSnw8vJCTEwMPDw8rJnN6chGI8706Ss6BpHi1Y24GZnRk1u157Qtefs1oPz8+1Y9Zrt2PTHKdybkRmV9rkSuLOKZoXAL9BQdQ7HaVK21Wi2GDh1qrSxOT3JzgzowEKaKCtFRiBRHliTUjpmDc2FjUFpsBBRWHAHA268c5VY+Zl5eCs7264wYfV9OoCFSCLWvRnQEReO4rJ25RUawPBL9iaxSo2biXJzzuxblpUag2DYLe1uDZLbNahIJx7cibHgH+Bf42+T4RGQ5lY87JLdWrWToMvjVsTP3iIuXOCJyRWY3DSqnPoqEmStxVD/4QnFUOF1Fls2Ovf3AapgiuQE2kWjqAF6CdzUcebQz94gI0RGIhDJ7+qDy+r8hHT2gqzICdY4xWcRNY0JlYY7tnkCWsS15NaZ3fwxypWN8TYickdqf5fFqWB7tzC2S5ZFck1nrj7LJjyBD3xl1VUbYat9pW/ELqoKuyLbXYdbWViC+eBNGaGdA1ivvmk8iV6D25/WOV8PyaGc8bU2uxhQYhtKJDyOjNgoN5SY4Wmn8g5t7kV2eJyfvFNL7dUZXQ29OoCESwI2nra+K5dHO3CPCRUcgsgtTaHsUj/87MqpCoS8xAbD9vtO21Fhrw1PWf3H0+E8IHR4NvwI/uz0nEV3A09ZXx/JoZ26X2BOcyJkY23VD4agHca7cH8YiMxy9NF4goyL/nF2fcfuB1bh12DNQFfD0NZE9ccLM1XG2tZ25h4UBEmdUkvMxdO6D7Dvfwv4eT+FsoS+MTnTNnl9wLfT1dXZ9Tlk2Y9vxVZAC3O36vESujiOPV8eRRzuTNBqoQ4JhKikVHYXIKvSxQ5E3+C5kFbpDzgec8UI9T+8yIc+r01Xgt5LNGO4znRNoiOxBAtR+LI9Xw/IogHtEJMsjObyGfqOQ2+c25BSofi+NzsukzxP23Nm5JxHRvws663sKy0DkKlS+Gkhqnh28GpZHAdwjItBw4oToGEStUj/kemTHzEBeAQAnL41/qC7NFPr8h5N/QMiIaPjma4XmIHJ2bjxlbRFe8ygA13okR1Q7fCZO37EKB7S/F0cX4emjR02pfZbpuZJtv62EOYK/solsiWs8Woa/iQRwj4oSHYHIIjIA3dg7ceK21TikmYTCQmeYOd0y2gBl7EUvy2ZsP/EBJH9OoCGyFXWAp+gIDoGnrQXw6NpVdASiK5JVatRMmItz/tde2HO62DEX9rYGCYWiIzSp0ZXhQPmPGOY5FbKBE2iIrM093Ft0BIfA8iiAR0yM6AhEl2R206B68oPI8ByAqnIDUOq6pfEP9VXnRUdo5nx2MiIGdEanqljRUYicjhvLo0VYHgVwj4iAytcX5poa0VGIAACyhxcqrn8Y6VJP6KoMQJ1BdCRFULmZUVGQJTrGRQ4lbUbIiMehzfcRHYXIqXDk0TK85lEQj27dREcgglnrj9Kbn8GhycuQVN3tQnGkJv5B1TAZlTn6uu3ASsgRatExiJyGOsADKg+OqVmC5VEQlkcSyewfgqLZL+Lg6P/geEUH1NUosyCJ5u5RIjrCZZnNJmw79SEkP77YEVkDRx0tx986gvC6RxLBFNIOxeMfRkZ1KPQlJjjHvtO2o6/PER3himqqS3Cocguu8ZgCcAINUZu4hfMyEEuxPAriEcORR7IfY7tuKBr9IDLK/GEsNoOl0TJVhedER7iqzPNJCB/YBR0r+YaUqC048mg5lkdBOPJI9mDo1BsFw+7DuWJvmAtkABydspRPQB3KMqtFx7DIwcSNCBkxHz75XqKjEDks9wiOPFqK1zwK4hYSAnVgoOgY5KT0sUOReecyxHd5DOkFXjCbZNGRHI63tlx0hBbZemAF5HBOoCFqFQlwD+ObL0uxPArESTNkbY19RyH9zncRHzUPmfnuMJtZGltLNjnWxt1mswk/n/4Iki9PKBG1lFuQJyR3vvmyFMujQDx1TdbSMHgyUuesQHzwHcjOV0FmZ2wzXVmW6AgtVlVdjMM12wF3/monaglOlmkZvkUViJNmqK3qhs9EVofrUVhoBgpEp3EeGk8jKgvzRMdolXOZxxA+qDM6VHAbVCJLcbJMy/DtqUAceaTW0o2dgxO3r8JBzaQLxZGsShtYCUcevj2QsAF1UQ2iYxA5DJbHlmF5FMijR09Axf8FZBlZpUb1pPuQdOtqHMZIlBRxuR1bcVMXiY7QZlsPcgINkaXc22lFR3AobC4CqbU+HH2kqzK7aVA59REkzFyJo4YhKC/lbjC21lCTLTpCm5lMRuw4s4YTaIiuQuXtBvdQjjy2BH+rCOY1cAAaU1NFxyAFkj28UHH935Eu9bqw53Qd9522B0mSUVGQKTqGVVRWFuJo4A4MdpsAGB33NDyRLWk6+omO4HA48iiY96BBoiOQwpi1/ii9+RkcmrwMSdUxF4oj2Y1fsA6GRue5XjA98why/ZyjDBPZAstjy3HkUTCvgQNFRyCFMPuHoGTSI8iob4+GCiMAnp4WwcOrRHQEq4s/9jVmjFwArzwP0VGIFMejA8tjS3HkUTBNdDTUoSGiY5BAppB2KLj9Vfw27F84VRqBhlqWRpGMesdcoudqthx4HwjjeAFRM2oJmmhOlmkplkcF8B7A0UdXZIzqgrw7Xkf8oBeQUhQEfQNnTytBVdE50RFswmTS4+fUNZC0LJBEf3CP0nJnmVZgeVQAnrp2LcaOvZA95y3E91qI1EI/GPVcp1EpvHwbUVtRJjqGzVRWFiChbhfgJomOQqQIHrzesVX4FlQBvAexPLoCffchyBtyF84XesBcIAPg7Fel8fEvR4XoEDZ2NuMgwoZ0QruyjqKjEAmn6egrOoJD4sijAnj26gXJgxeyO6vGvqOQcec7iG93HzLzNTCbWRqVSpJdY4/H/Ue/QkM7vegYRMJ5dPQXHcEhsTwqgKTRwLN3b9ExyMoaBk9G6p0rEB98B87nqx15tzuXUVtxXnQEu9lycAUn0JBLUwd6QO2nER3DIbE8KgRPXTuPuuEzcfqOlfjN9ybk5YtOQ5ZSu5tQWej4O8tYymjUY+fZTziBhlwWr3dsPf7WUAhOmnF8ujF3IDNi7IU9pws5CcbR+AdXobbYtWa8l1fkIyH4VwxUjwFMHBon18LFwVuP5VEhvAYNAiQJPLfpWGSVGjUT7sW5gOtQXmIEilyrfDgTN/di0RGEOJt+ABFDOyOyNFp0FCK70nTi9Y6txdPWCuEWGAjPXr1ExyALmd00qJryMBJmrsRRw5ALxZEcmr4uV3QEYfYe+QIN7bgNJrkOlZ8Gmkgf0TEcFsujgmjHjBYdga5C9vBC+fQFODbtPRyr74uqcr7gOgMZMiryM0THEGrLwfeAUJ6MItfg2T1QdASHxvKoINrRLI9KZfbxQ+nNi3Fo8jIk1cSgppKl0Zn4Btahsa5WdAyhjEY9fkn/FJIPCyQ5P8/YINERHBp/SyiIZ79+UAcGwlTh7MsUOw6zfwhKJz2C9Pr2aKgwAuDpaWfkrXXeXWVaoqw8F8khe9BPPZITaMh5qSV4xgSITuHQWB4VRFKp4DNiBKp/+kl0FJdnCmmHkgl/R3pVGPSlJrA0OjeTIU90BMVIObsfYdd0QkRJO9FRiGxC08EPKk/Wn7bgaWuF4XWPYpkiOyPvjtcRP+gFnC4Khr6Bs6ddQU1plugIirLn8OdobMc3TOScvHrwese2YvVWGJ+RIwGVCjBznUB7MnbshfwR9yGz2AemQhkAv/6uwsPbgKo819iWsCW2HFqBWQOfBriSADkZXu/Ydhx5VBi3wEB49e0rOobL0McMQtadb2N/18eRnu8Nk5HXebka30BeY3wpBkMDdp37DJI3xxjIeaj9PeAewSV62oq/FRTIZ/Qo1Ccni47h1Br7jERuv9uQXaCGnA8ALI2uSpIKRUdQrNLSbBwP3oe+quGAmT8j5Pg8Y3nK2ho48qhA2tFjREdwWg2DJuHsnSsQHzIH5/PV3NCHUF/tOvtZt8bp1L0oDuEm7eQceMraOlgeFcizT2+oQ0JEx3AqdcNuQsqclfjNbyZy+TpIv5PUZlTkZ4mOoXi7D62Dvh0nj5GDU0vw4BI9VsHyqECSJEE7cqToGE6hdswdOHH7Khz0mIyCAk6Coeb8g2pgMuhFx3AIWw6tgBTiLjoGUat5dPaHSqMWHcMpsDwqFJfsaT1ZpUbNxHlInr0ah6RRKCniiAldmsazRHQEh6E31GNX1ueQvPniS46Jp6ythxNmFEo7ejQkLy/I9fWiozgMs5sGNZPuR7rXoAt7TnOJEboKQ0Ou6AgOpaQkCyeC4tFHNYwTaMixSIBX32DRKZwGRx4VSuXjA99xY0XHcAiyhxfKpy/AsWnv4Vh93wvFkcgClYXnREdwOKdS96AkhDPUybFoOvrBLcBTdAynwfKoYH433ig6gqKZffxQevNiHJq8DEk1MaipZGkky3n71aO+ulJ0DIf066G1METxGmJyHN79Q0VHcCosjwqmHTUKKn9/0TEUx+wXjOJbX8DBsf+H4xUdUVfD09PUcj5+5aIjOLSfjqwAOIGGHIFKglc/lkdrYnlUMEmjge+kiaJjKIYpOBKFty3BbyNew8nSSDTUsjRSG5i5JWFb6PV12H1+PSQvTqAhZfPoFgC1D9/oWBPLo8L5T5smOoJwpsjOyLvjdcQPfgmni0Ogr+fsaWo7XXmW6AgOr7g4EyflA3wlIUXzHsBRR2vjbGuF877mGriFhsJY4npLihg79EDByAdwrtgHpkIZAK+xIutw9zChspAzra3hZMpuhF/XGSFFfIEm5ZHcVfDqzVnW1sb3iwonqVTwmzpFdAy7MnQbiKw738b+bvORlu8Nk5FLgpB1+QZVQpb5ZsRadh1cA0MUf05JeTx7BEHlwXEya2N5dACuMuu6sc9IZNz5DvZHP4hz+RqYuY4c2YibW5HoCE5ny7EVkIJ5XRkpC2dZ2wbLowPw6tcP7h07iI5hMw2DJuLsnBWID5mD8/lqyOyMZGMNNdmiIzidxoZaxOV8yQk0pBiSpxqePbirjC2wPDoIv6lTRUewurrrZiBlzkr85nczcjnxlexFklFRkCk6hVMqLMrAaRziKwspglfvEEhu/Ga0BX5VHYQzzbrWjbkdJ29fhYOe16OggNedkX35BdfC0MBtP23l+OldKAsrFR2DiLOsbYjl0UF4dO0Kj549RcdoNVmSUDNxLpJnr8ZhaTSKi7jcDonh6eV6KxfY2y8HPoYxSnQKcmUqX3d4dA0QHcNpsTw6kIBbbhEdocXMbhpUTfk7EmetxhHjNSgr4cLeJJZJny86gkvYcmwFpCBOoCExfAaHQ1JJomM4LUmWOT3BUZh0OqSPHgNzXZ3oKFclazxRef3fka7uzT2nSVHcpM+gK+fooz1ERsRgTOBsyA0800B2JAERi4fCLdBTdBKnxZFHB6LWauE3fbroGFdk9vFD2c2LcfiG5Uis6c7iSIri5dPI4mhHBYVpOKM6CnAAiOzIMzaIxdHGWB4dTOCdc0RHuCSzXzCKb30BB8f+H5IrOqK2mqenSXl8AipER3A5Sad2oCKiXHQMciE+wyJFR3B6LI8OxjM2Fl6DBomO0cQUHImC25bgtxGv4WRpJBpqWRpJuSQUio7gknb89iEn0JBdqIM84dk9UHQMp8fy6IAC54gffTRFdkbeHf9G/OCXkFIcAn09r2ki5aurOi86gsvakrCSE2jI5rTXRkKSeJ2ErXHCjAOS9XqkjRsPU1mZ3Z/b2KEH8kc8gMwSH+45TQ5F7WZCffkKmE0cHRelXVQsRvndArmRbzbJBtxUiHzuGqh9+CbF1jjy6IAkjcbuy/YYug1E1p1vY3/MfKQXeLM4ksPxC65hcRQsLz8Vqe4JnEBDNuHdL4TF0U5YHh1U4B23Ayrb/+9r7D0cGXe+g/3RD+JcvgZmE0sjOSZ3TbHoCAQg8cR2VEZUio5BTsjnOk6UsReWRwflHhUF7ZgxNjt+w8AJOHvnCsSH3YXz+Wrw4gZydPq6HNER6Hc/H/gApigOP5L1uLfTwqODn+gYLoPl0YHZYtmeuuumI2XOSvzmPwu5+QBYGskJyJBRWXhOdAz6gyxja+L7kAJ5ipGsQ8tRR7tieXRgPiNHwr1DB6scSzf6Npy8fRUOet6AggKzVY5JpBS+AfVo0NWIjkF/UldXjX1F30Hy4MsQtY3k6QbvAaGiY7gU/tQ6MEmS2rRsjyxJqJk4F8mzV+OwagyKizgDkpyTt6/9Vyagq8vLS8FZTTIn0FCb+AwJh+SuFh3DpbA8OriA2bOh8mvZdR6y2g1VN/wNibNW44jxGpSVcAYqOTezMV90BLqMhONbURVRJToGOSq1BO3IdqJTuByWRwen1vpYfO2jrPFExbT5ODr9fRxr6I/KMu47Ta6hpixTdAS6gu0HVsMUyeFHajnv/qFwC/AQHcPlcJFwJ2AsL0f6+AmQGxou+XGzjx/KJ/0dGeZu3HOaXI7G04jqwnfBJQOUzccnENO7Pwa5km9qyUISEP7kYLiHeYtO4nI48ugE3IKCEDBr1kW3m/2CUXzL8zg49v9wvLITiyO5JL+gChZHB1BbW4H44k2QNHxZIst49gxmcRSEP6VOIviB+wE3NwCAKTgShbctwW8jXsPJsig01LI0kutSqQpFRyAL5eSdQrrHCU6gIYv4jm0vOoLLchMdgKzDvV07+Nx5H842dsa5ykAYis0AOHuaqL4mW3QEaoGjx39C6PBo+BVwwWe6PI8u/lwUXCCOPDoR7X1/x9lifxgauU4jEQBIKjMq8jlZxtFsP7Aa5ki+PNHl+Y6PFh3BpfGn04kERfqg66Aw0TGIFMMvSAejXi86BrWQLJux7fgqSAHcgYYupunoB89ugaJjuDSWRyczZGonXi9E9DsPr1LREaiVdLoK/FaymRNo6CJ+E6yzsxq1Hn8qnUxwOy26cpsmIgCAsTFXdARqg+zckzjneUp0DFIQTQdfeHbnqKNoLI9OaMiNnTj6SASgsuic6AjURoeTf0BNpE50DFIIXwWMOs6bNw8zZ84UHUMolkcnFNLeF537hYiOQSSUt28D6irLRccgK9h2YCXMEXy5cnXu7bXwig0SHQPLly/H2rVrRcdoEhcXB0mSUFlZabfn5E+jk7pmehdIHH0kF+bjz+LoLGTZjO0nPoDkzwk0rsx/cifREQAA/v7+CAgIEB1DKJZHJxXSXovYYZGiYxCJIxeITkBWVKMrw4HyHyG582XLFXnEBNj1WscNGzagb9++8PLyQnBwMCZOnIja2loAzU9br1u3DsHBwWhsbGz2+JkzZ+Kee+4BAGRkZOCmm25CeHg4tFothg4dil9++aXZ/d9//33ExMTA09MT4eHhuPXWW5s+Zjab8frrr6Nz587w8vJC//79sWHDBgBAVlYWxo0bBwAIDAyEJEmYN2+eLb4kzfCn0IldO70L3DhTkVyUrjxLdASysvPZycj0ThEdg+xNAvyndrHb0xUUFGDOnDm4//77kZKSgri4OMyaNQvyJbY5nT17NkwmE3744Yem24qLi7Flyxbcf//9AACdToepU6di165dSExMxA033IDp06cjO/vCBgZHjx7F/Pnz8c9//hOpqanYvn07Ro8e3XS8119/HevWrcOqVatw6tQpPPnkk7j77ruxZ88eREdH47vvvgMApKamoqCgAMuXL7fllwcAIMmX+mqQ0zj0wzkc3ZolOgaRXbm5m1Bb+i5kMxfMd0Y3jngc2nwf0THITrwHhyNodne7PV9CQgIGDx6MrKwsdOzY8aKPz5s3D5WVldi8eTMA4NFHH0VWVha2bt0KAPjvf/+LFStWID09HdJlrh/r06cPHn74YTz++OPYuHEj7rvvPuTm5sLX17fZ/RobGxEUFIRffvkFw4YNa7r9wQcfRF1dHb744gvExcVh3LhxqKiosNvpdA5LObmBkzvAy08jOgaRXfkFV7E4OrFtB1ZCjlCLjkF2ILmr4D/54gJnS/3798eECRPQt29fzJ49Gx9++CEqKioue/+HHnoIO3bsQF5eHgBg7dq1mDdvXlNx1Ol0WLhwIXr27ImAgABotVqkpKQ0jTxOmjQJHTt2RJcuXXDPPfdg/fr1qKurAwCkp6ejrq4OkyZNglarbfqzbt06ZGRk2PgrcXksj05O4+mGa6Z1Fh2DyK7c3ItERyAbMptN2HbyA0j+bqKjkI1pR7WD2t/Drs+pVquxc+dObNu2Db169cK7776L2NhYZGZeeqvTgQMHon///li3bh2OHTuGU6dONbvucOHChdi0aRP+/e9/Y9++fUhKSkLfvn2h/333K19fXyQkJODLL79EZGQkXnrpJfTv3x+VlZXQ6S4sU7VlyxYkJSU1/Tl9+nTTdY8isDy6gF4jIhEY4S06BpHdNOpyREcgG6upKcWhii0AJ9A4LZXWHb5j2gt5bkmSMGLECCxZsgSJiYnQaDTYtGnTZe//4IMPYu3atfjkk08wceJEREf/b+/t+Ph4zJs3DzfffDP69u2LiIgIZGVlNXu8m5sbJk6ciDfffBPHjx9HVlYWfv31V/Tq1QseHh7Izs5Gt27dmv354zk0mgtnF00mk/W/EJfBt20uQKVWYdisbtj6/nHRUYjsQEZFARcHdwWZ55MQPrALOlbGiI5CNuA3sQNUHvavKYcOHcKuXbswefJkhIWF4dChQygpKUHPnj0v+5g777wTCxcuxIcffoh169Y1+1hMTAw2btyI6dOnQ5IkvPjiizD/6bKan376CefOncPo0aMRGBiIrVu3wmw2IzY2Fr6+vli4cCGefPJJmM1mjBw5ElVVVYiPj4efnx/mzp2Ljh07QpIk/PTTT5g6dSq8vLyg1Wpt9vUBOPLoMjr3C0G77gGiYxDZnF9wLfT1daJjkJ0cTNyI2qh60THIytxCveAzVMxyc35+fti7dy+mTp2K7t2744UXXsDSpUsxZcqUyz7G398ft9xyC7Ra7UW7z/z3v/9FYGAghg8fjunTp+P666/HoEGDmj4eEBCAjRs3Yvz48ejZsydWrVqFL7/8Er179wYAvPrqq3jxxRfx+uuvo2fPnrjhhhuwZcsWdO584ZK0du3aYcmSJXj22WcRHh6Oxx9/3PpflL/gbGsXUny+Gt++cRTg/3FyYmHRWcg+vlF0DLIjlUqNW695BlKR/U7bkW0F39sLXr2CRcdokQkTJqB379545513REexOY48upCwjn7oPjRcdAwimzIZ8kVHIDszm034+fRHkHx5JZYz0HT2d6jiWFFRgU2bNiEuLg6PPfaY6Dh2wZ80F3PtTV2QkVgCk4HLmJBzqi659IxIcm5V1cU4XLMdQ90nA/z95rgkIOBGx1ohZODAgaioqMB//vMfxMbGio5jFyyPLsYv2AsDJ3XgwuHklDx8DKjK5TI9rupc5jGED+qMDhVdRUehVvIeGAZNe9+r31FB/jpz2hXwtLULGjylI/zDvETHILI634By0RFIsAMJG1AX1SA6BrWCyscN/jfabxtCaj2WRxfk5q7G2DtdY2idXIuEQtERSAG2HlwBOZw70Dga/6ldoPZxFx2DLMDy6KLa9whC7HURomMQWVV9dbboCKQAJpMRO86s4QQaB+LRLQA+gzmh01GwPLqwEbd2gyff5ZGTUKnNqMjnZBm6oLKyEEdrdgBukugodDVuKgTM7CY6BbUAy6ML89JqMPwW/sCSc/AProHJaBQdgxQkPfMIcv34hkLp/MZFwz2E1+E7EpZHF9dzeCTaxQaIjkHUZu4exaIjkALFH/sa9VGNomPQZbiFecN3rJj9q6n1WB4JY+/sAbUbvxXIsRnqc0VHIIXacvB9IIzXPyqOBATO6gZJzdcfR8P/Y4SAcG8MntJRdAyiNqkszBAdgRTKZNLj59Q1kLQskEriMzQCHp38RcegVmB5JADAoOs7IjDCW3QMolbx8a9HfU216BikYJWVBUio28UJNAqh8nWH/xTH2kmG/oflkQAAajcVxt7dA+DvVXJAPr5loiOQAzibcRB5/lmiYxCAgGldofLiSLCjYnmkJlHdAtBrRJToGEQtZjbli45ADmL/0a/Q0E4vOoZL84wNhHf/UNExqA1YHqmZEbd0g2+wp+gYRC2iK88SHYEcyJaDKziBRhCVtxsCZsWIjkFtxPJIzWi83DDpvl6QVDx/TY5B42lEZVGe6BjkQIxGPXae/YQTaAQIuLkb3Pw9RMegNmJ5pItEdgvA4Bs4+5ocg29gJSDLomOQgymvyEdC/a+Amm+U7cV7cDi8+/J0tTNgeaRLGnpjJ4R39hMdg+iq1Ooi0RHIQZ1NP4CCQO6Hbg/qYE8EzOgqOgZZCcsjXZJKrcKk+3vB3UMtOgrRFTXo+OJPrbf3yBdoaGcQHcO5qSQE3R4LFV9PnAbLI12Wf6g3Rt3OC5tJuSRJRkU+9y6mttly8D0glNc/2orfhA7w6MAzWc6E5ZGuqOfwKHQdxGtUSJn8gnUwNDaIjkEOzmjU45f0TyH5sEBam6ajH3zHRYuOQVbG8khXNfauHtAGcnYcKY+Hd6noCOQkyspzkdQYxwk0ViR5qBF0eyxX73BCLI90VZ4+7pgwrxck/vyTwhgbckVHICdyJi0ehUH8nrKWgJnd4BbEdYOdEcsjWaR9bCAGTOwgOgZRM1XFvN6RrGvP4c/R2M4oOobD8+ofCp+BYaJjkI2wPJLFrr2pC0KitaJjEAEAvHwbUVvB09ZkfVsOreAEmjZQB3gg8OZuomOQDbE8ksXUbipc/2AfaDy53AKJp/UrFx2BnJTB0IBd5z6D5M0C2WJqCUFzekDlya+dM2N5pBYJCPfGxPt7A7z+kYQrFB2AnFhpaTaOG/cBnOzRIgHTu8KjI5flcXYsj9RinfuFYOjUTqJjkIurrcgSHYGc3OnUvSgOyRcdw2H4DI2A9rpI0THIDlgeqVWGTuuMTv1CRMcgF6V2N6GykDvLkO3tPrQO+nYm0TEUTxPti4CbuP2gq2B5pFaRJAmT7uuFgHBv0VHIBfkHVcFs4gs62ceWQysghbiLjqFYKq07gu/uCcmNlcJV8P80tZrGyw1THu4Ld06gITtz05SIjkAuRG+ox66szyF583fdRdQSgu/uCbU/N5JwJSyP1CZBkT6YOLcXJ9CQXTXW5oiOQC6mpCQLJ4zxnEDzFwHTusCjk7/oGGRnLI/UZl0GhmLwDR1FxyAXIUNGZeE50THIBZ1K3YOSEM7y/4P34HBoh0WJjkECsDySVVw7vQs69gkWHYNcgG9QPRprdaJjkIv69dBa6NuZRccQzr29lguBuzCWR7IKSSVh0v294B/qJToKOTlvb+4qQ2JtObwCcOEJNCqtO4Lv6cUJMi6M/+fJajy83THlkb5w9+BF5WQ7JiPX3SOx9Po67D6/HpKXC/6uU0kIvrMH3DhBxqWxPJJVBUdpMfnB3lDxonKykZrSTNERiFBcnImT8gGXexUNnNUNHl0CRMcgwVzs257soVPfEIy9O1Z0DHJCHt4GVJcUiI5BBAA4mbIbpaGus2yU36SO8BkSIToGKQDLI9lEz+FRuGZ6Z9ExyMloAypERyBqZtfBNTBEyaJj2JzPtRHwm9BBdAxSCJZHspmhN3ZG71FcxoGsR60qEh2B6CJbjq2AFOy8E2g8ewUj4CbOrKb/YXkkmxo9Jxad+3MPbLKO+urzoiMQXaSxoRZxOV865QQaTUc/BM+JhcTr2OlPWB7JplQqCZMf6I2ILtyBgNpGUptRnp8lOgbRJRUWZeA0DjnVq6pbmBdC5vaC5O58pZjaxom+zUmp3DRq3PhYPwRGeIuOQg7MP7gGJoNedAyiyzp+ehfKwpxjHVKVnwYh9/eBytt5T8dT67E8kl14+rhj2j/6w9tfIzoKOSiNh+vMaiXH9cuBj2F08Eu9JQ81Qub1hluAp+gopFAsj2Q3fsFemP6P/tB48hQItZyhIU90BCKLbDm2AlKQg47YqSUE39sLmiit6CSkYCyPZFch7X0x5ZF+ULnx4mtqmarCDNERiCzS0KDDnryvITnaG2UJCLotFp5dA0QnIYVjeSS7ax8biMn3cxcaspyPfz3qqitFxyCyWEFhGlJURwEH+jUXMKMrvPuHio5BDoDlkYToOigMkx5ggSTLePuVi45A1GLJp3agPNwxvncDZnSFdpiDX6xJdsPySMJ0G8wCSRYycUtCckw7D3yo+Ak0ATO6Qjtc4SFJUVgeSSgWSLKErjxLdASiVtuSsFKxE2hYHKk1WB5JOBZIuhJ3jRGVhbmiYxC1WkN9NfYVfAvJQ1kTaFgcqbVYHkkRWCDpcnyDqyDLZtExiNokLz8Vqe4JiplAw+JIbcHySIrBAkmX4uZWJDoCkVUkntiOyohK0TEQML0LiyO1CcsjKQoLJP1Voy5HdAQiq/n5wAcwRYn7/RYwvQu0I9oJe35yDiyPpDgskNREklGef050CiLrkWVsTXwfUqD9J9CwOJK1sDySIrFAEgD4BtXC0FAvOgaRVdXVVWNf0XeQPOz3EsziSNbE8kiK1W1wGG54uC/cNPw2dVVe3qWiIxDZRF5eCs5qkm0/gUYlIfCWGBZHsiq+KpOide4XgpsWDISnjzLXSCPbMurzREcgspmE41tRFVFls+NL7ioE39sLPkMjbPYc5JpYHknxIrr445bFg+Eb7Ck6CtlZdQmvdyTntv3AapgirT/8qPJxQ+jf+sGrR5DVj03E8kgOISDcG7csHoyQaK3oKGQnXj6N0JWViI5BZFuyjG3JqyEFWO/sijrIE6GPDIAm2tdqxyT6M5ZHchg+/h64+elBaN8jUHQUsgOfgArREYjsora2AvHFmyBZ4fpu93ZahD3SH+4hXlZIRnRpLI/kUDSebpj2j/7ofk246ChkYxIKRUcgspucvFNI9zjRpgk0HjEBCP1bP6h9NdYLRnQJLI/kcNRqFSbe1wsDJnUQHYVsqK7qvOgIRHZ19PhPqI6obtVjvQeGIWReb6gUtn82OSeWR3JIkiRhxC3dMHJ2jGL2iiXrUbuZUJHP8kiuZ/uB1TBHtuylWTumPQJv6w5JzZd0sg9+p5FD6z8hGpMf6A21G7+VnYlvcDXMJqPoGER2J8tmbDu+yrIJNNKFxb8DpnSGJPFdNNkPX3HJ4cUMCceMJ/rDU8u1IJ2FRsNZ1uS6dLoK/Fay+YoTaCSNGsF39eTi3yQEyyM5haiYQMx+bgiC23MpH2egr8sRHYFIqOzckzjneeqSH1MHeSLs0f7w6hNi51REF7A8ktPwC/bCLYsHo+ugMNFRqI0qC7k4ONHh5B9QE6lrdptHV3+EPTYA7hE+glIRsTySk3HXqHHD3/rg2hmdOZHGQWkD6tGgqxEdg0gRth1YCXPEhZdq7fAohNzfF2pu10qCsTySUxoytTOmPtwXGk8uW+FovHxLRUcgUgxZNmNn6ifwv7UrAmZ0haTmu2ISj+WRnFbn/qGY/dxQBEXx9I4jkY0FoiMQKYZvSChuXPgMfIdEiY5C1ESSZVkWHYLIlgyNJuz+LAVpR4tFRyELeLh/harifNExiITr2G8gbpy/CF6+fqKjEDXD8kguI3lXDn77Lh1mM7/llcrdy4CagvcA/loiVyZJuHbmbIy47W5IKp4gJOVxEx2AyF76T4hGaAdf/PzhSdRV60XHoUvwC6xETT6LI7kuD28f3PDok+g29DrRUYguiyOP5HLqqvX49bMUnD9RJjoK/UVo+5PIObFDdAwiIaJie2Hq40/DPyxcdBSiK2J5JJd1Ii4Xv32XDqPBLDoK/c7XfytKss6IjkFkVyq1Gtfdcgeuvfk2qFRcIYKUj+WRXFp5QS12rjmF0hzd1e9MNiVJMgy6FTDqeUkBuY6AiEhMfXwhImNiRUchshjLI7k8k9GMQ9+fQ+Iv2QB/GoTxD61G0dmPRMcgsps+4yZh3Ly/QePpJToKUYtwwgy5PLWbCsNv6YYOvYOw69MU6CoaRUdySR6eJaIjENmFp9YXk/72OLpfO0J0FKJW4cgj0Z801BoQt/4MMhJYZOwtKGwv8lOPio5BZFMd+vTDDY89Bd+gENFRiFqN5ZHoElJ+K8C+r8/C0GgSHcVlqOW1qK0sFx2DyCbUbm4Ycce9GDLtZkgStxgkx8bySHQZVSV12LnmNIoyq0VHcXpevg2oyH5fdAwimwhqF40b5y9CWKcuoqMQWQWXrncScXFxkCQJlZWVoqO02Nq1axEQECA6xkX8Q70xa9FgjJwdA3cPLp9hS1r/CtERiKxO7eaG626Zg3v+8w6LIzkVTpgh4W6//XZMnTpVdIxLUqkk9J8QjS4DQ7H3q7PIOl4qOpJzMnMva3Iu7Xr0xqSHHkdw+2jRUYisjuWRhPPy8oKXl7KXqvAN8sSNj/ZDRmIx9n11FrVVXIvQmmorz4uOQGQVnj5ajLrrPvQdP5nXNpLT4mlrB2E2m/H666+jc+fO8PLyQv/+/bFhw4bL3r+urg5TpkzBiBEjUFlZCbPZjH/+859o3749PDw8MGDAAGzfvr3p/llZWZAkCRs3bsS4cePg7e2N/v3748CBA82O+91336F3797w8PBAp06dsHTp0mYf79SpE/71r3/h3nvvhVarRceOHfHDDz+gpKQEN910E7RaLfr164ejR/83q1app60vpevAMNz5ynXoO7Y9+LpgHW7uJlQUZIuOQdRmPUaMwX1vr0K/CdezOJJTY3l0EK+//jrWrVuHVatW4dSpU3jyySdx9913Y8+ePRfdt7KyEpMmTYLZbMbOnTsREBCA5cuXY+nSpXjrrbdw/PhxXH/99ZgxYwbS0tKaPfb555/HwoULkZSUhO7du2POnDkwGo0AgGPHjuG2227DHXfcgRMnTuCVV17Biy++iLVr1zY7xttvv40RI0YgMTERN954I+655x7ce++9uPvuu5GQkICuXbvi3nvvhaPO1dJ4uWH0Hd0xa/FgBLfTio7j8PxCqiCbuUUkOS7/8Ajc8twS3Dh/Ebz9A0THIbI5zrZ2AI2NjQgKCsIvv/yCYcOGNd3+4IMPoq6uDl988QXi4uIwbtw4pKSk4Pbbb0dMTAy++OILaDQaAEC7du3w2GOP4f/9v//X9PhrrrkGQ4cOxYoVK5CVlYXOnTvjo48+wgMPPAAAOH36NHr37o2UlBT06NEDd911F0pKSrBjx46mYyxevBhbtmzBqVOnAFwYeRw1ahQ+++wzAEBhYSEiIyPx4osv4p///CcA4ODBgxg2bBgKCgoQERGBtWvXYsGCBQ452cdsMiPplxwc2ZIJo54FqDXColOQfXyb6BhELaZSqzF42s0YduscuGs8RMchshuOPDqA9PR01NXVYdKkSdBqtU1/1q1bh4yMjGb3nTRpErp164avv/66qThWV1cjPz8fI0Y0381gxIgRSElJaXZbv379mv47MjISAFBcXAwASElJueQx0tLSYDKZLnmM8PBwAEDfvn0vuu2P4zoylVqFQdd3xJyXrkWHXkGi4zikxtpc0RGIWiwyJhZ3v7Eco++cx+JILocTZhyATqcDAGzZsgXt2rVr9jEPj+a/tG688UZ89913OH36dLPCZil3d/em//7jmh1zC08pXuoY1jiukvmFeGH6/AFIP1aMA5vSUV3aIDqSg5BRkZ9x9bsRKYRPQCCG33YX+o6bDEnF8RdyTSyPDqBXr17w8PBAdnY2xowZc8X7vvHGG9BqtZgwYQLi4uLQq1cv+Pn5ISoqCvHx8c0eHx8fj2uuucbiHD179kR8fHyz2+Lj49G9e3eo1VwHEQC6DQ5D5/4hOBGXi6PbstBYaxQdSdF8g2pRklEnOgbRVbl7eGLwtJsxdMYsaDyVvToEka2xPDoAX19fLFy4EE8++STMZjNGjhyJqqoqxMfHw8/PD3Pnzm12/7feegsmkwnjx49HXFwcevTogUWLFuHll19G165dMWDAAHzyySdISkrC+vXrLc7x9NNPY+jQoXj11Vdx++2348CBA3jvvffw/vvcGeTP1G4qDJjYAT2GReLYtiyciMuDyeg8o6zW5OVTJjoC0RVJkgq9x07EiNvvhjaQl6YQASyPDuPVV19FaGgoXn/9dZw7dw4BAQEYNGhQswkwf/b22283K5Dz589HVVUVnn76aRQXF6NXr1744YcfEBMTY3GGQYMG4ZtvvsFLL72EV199FZGRkfjnP/+JefPmWemzdC6ePu4YcWsM+o5tj4ObM5B2rBjg9LRmTIY80RGILqvzgMEYfdd9COnQSXQUIkXhbGsiOynKqsZv36UjP61SdBTF0KjXo7q0SHQMomZCO3XBmLvuR8d+A0RHIVIklkciOzuXVIIDmzJQWeTa1/p5+BhQlfuu6BhETbTBIRh5+z3oNWocJ8MQXQHLI5EAZpMZp/bl48iWTNTXGETHESI4qgh5pyy/5pbIVjReXrjmptkYdONNXHaHyAIsj0QCGRpNOLUvD4k7s1HnYvtlh7U/juwTv4iOQS7M00eLgVOmY+CUGfDS+oqOQ+QwWB6JFMBkMCPlt3wk/JyNmnLXWCNS6/sjSrPTrn5HIivz9g/A4BtnYsDkqdB4eYuOQ+RwWB6JFMRsMuPs4SIc237eqa+JVKnNaKxaAZPBNU/Zkxi+waEYMn0W+k6YzNPTRG3A8kikQLJZRnpCMY5tP4+yXJ3oOFYXEFaFwtSPRccgFxEQEYlrbpqNXqPHQ+3GFeqI2orlkUjhMo+X4ti2LBRlVouOYjVh0enIPv6D6Bjk5ILbd8C1N9+G2OGjoFJxFywia2F5JHIQOWfKcWxbFvJSK0VHabPAkN0oSEsUHYOcVHiXbrj25tvQbegwSJIkOg6R02F5JHIwpbk1OLEnD2cPF8HYaBIdp1Uk48eor6kSHYOciNrNDTHXjsCAyTeiXY9eouMQOTWWRyIH1VhvxJkDBTi1Nw8VhY4zucbHvx5lWStFxyAn4Rcahn4TbkDfCdfD289fdBwil8DySOQEcs+U4+SePGQml8JsVvaPdGj7XOSc+EZ0DHJkkoRO/QdhwOSp6DJwKHeDIbIzTjsjcgLtewShfY8g1FY24tS+PJzen49ahS46LpsLREcgB+Xp64c+Yyei/8QpCIiIFB2HyGVx5JHICZlNZpxLKsXJvbmKm2Dj6fENKgtzRccgBxLRrTsGTL4RscNGwU2jER2HyOWxPBI5uYrCWqQeKkTakSJUl4rdvUbjaUR14bsAf+3QVWgDgxA7fBR6jhyH8C7dRMchoj9heSRyIYWZVUg7XIT0Y8Woq7b/ae3gyFLknV5n9+clx+Dpo0XMdSPQY/gYRPfqw2sZiRSK5ZHIBZnNMvJSK3D2SBHOJZZAX2+0y/OGtT+F7BM/2+W5yDG4eXig6+Br0WPEGHQeMAhqN3fRkYjoKlgeiVycyWDG+ZNlOHukEFknymAymG32XH4B21GcedpmxyfHoFK7oVP/gegxfDS6DR0Gd09P0ZGIqAVYHomoib7BiHNJJUg7UoS81EqYjFYskpIMU+1KGBrFXndJYqjUakTF9kSP4aPR/bqR8PL1Ex2JiFqJ5ZGILsnQaELumXKcP1mG86fKoCtvbNPx/EN0KEr7wErpyBF4+weg84DB6DxwCDr2GwhPH63oSERkBSyPRGSRsnwdzp8sQ/bJMhRkVMFsatmvjrDoTGQf32SjdKQEkqRCRLcYdB44BF0GDkVY567cW5rICbE8ElGL6euNyPl9VDL7ZJlFC5IHh+9H3pnDdkhH9uTl64dO/Qeh88Ah6NR/EE9HE7kAlkciarPS3BqcP1mG3DMVKMyshrHRdNF91FiH2opSAenImtTu7ojoGoPo3v3RZeAQRHSN4ZI6RC6G5ZGIrMpsMqMkW4f89EoUpFeiIL0KkkqHiuwVoqNRK2i8vBEV2xPte/RGux69ENEtFm7uXE6HyJWxPBKRTcmyjIqCCuSmHEFBWioK0s6gPC8Xsmy7JYGo9QIjoxDZLRaRMT0Q2b0HQjt2gkqlFh2LiBSE5ZGI7K6xrg6F6WdRkJ6KwoyzKDmfherSYm5baGfa4BCERHdERNcYRMbEIrJbLK9ZJKKrYnkkIkXQN9SjLDcbZTnZKM3NRlnOeZTmZkNXxusk28rbPwAh0R0QHN0RIe07Xvg7ugM8vH1ERyMiB8TySESK1lhXi9KcP8rk+Qt/52SjrqpSdDTF8dT6IiS6I4Lbd7jwd/SFvzmaSETWxPJIRA6psa4ONWUlqCkr/dOf//1bV1bqVLvZuLlroA0Ohm9wKHyDQ37/u/m/PbVchJuIbI/lkYicVoNOd1HBrK+pQmNdHfT1F/40/XddHRrr6yCbbT+RR5JU0Hh5QePlDQ9vb2i8vKHx9obH7397aX2hbSqIIfANDoG3n7/NcxERWYLlkYjoTwwNDWisq0Xjn8qlob4eZvPFa1dejaRSQeP5R0n0gcbbCx5e3nD39OLOK0TksFgeiYiIiMhi3BaAiIiIiCzG8khEREREFmN5JCIiIiKLsTwSERERkcVYHomIiIjIYiyPRERERGQxlkciIiIishjLIxERERFZjOWRiIiIiCzG8khERIqTlZUFSZKQlJQEAIiLi4MkSaisrAQArF27FgEBAcLyEbkylkciIlK84cOHo6CgAP7+/qKjELk8N9EBiIiIrkaj0SAiIkJ0DCICRx6JiKgVNmzYgL59+8LLywvBwcGYOHEiamtrAQBHjhzBpEmTEBISAn9/f4wZMwYJCQnNHn/mzBmMHDkSnp6e6NWrF3755RdIkoTNmzdf8vn+etr6r0pKSjBkyBDcfPPNaGxstOanSkR/wfJIREQtUlBQgDlz5uD+++9HSkoK4uLiMGvWLMiyDACoqanB3LlzsX//fhw8eBAxMTGYOnUqampqAAAmkwkzZ86Et7c3Dh06hA8++ADPP/98q/Pk5ORg1KhR6NOnDzZs2AAPDw+rfJ5EdGk8bU1ERC1SUFAAo9GIWbNmoWPHjgCAvn37Nn18/Pjxze7/wQcfICAgAHv27MG0adOwc+dOZGRkIC4urulU9GuvvYZJkya1OEtqaiomTZqEm2++GcuWLYMkSW34zIjIEhx5JCKiFunfvz8mTJiAvn37Yvbs2fjwww9RUVHR9PGioiI89NBDiImJgb+/P/z8/KDT6ZCdnQ3gQuGLjo5udg3jNddc0+Ic9fX1GDVqFGbNmoXly5ezOBLZCcsjERG1iFqtxs6dO7Ft2zb06tUL7777LmJjY5GZmQkAmDt3LpKSkrB8+XL89ttvSEpKQnBwMPR6vVVzeHh4YOLEifjpp5+Ql5dn1WMT0eWxPBIRUYtJkoQRI0ZgyZIlSExMhEajwaZNmwAA8fHxmD9/PqZOnYrevXvDw8MDpaWlTY+NjY1FTk4OioqKmm47cuRIizOoVCp89tlnGDx4MMaNG4f8/Py2f2JEdFUsj0RE1CKHDh3Cv//9bxw9ehTZ2dnYuHEjSkpK0LNnTwBATEwMPvvsM6SkpODQoUO466674OXl1fT4SZMmoWvXrpg7dy6OHz+O+Ph4vPDCCwDQ4lPParUa69evR//+/TF+/HgUFhZa7xMloktieSQiohbx8/PD3r17MXXqVHTv3h0vvPACli5diilTpgAAPv74Y1RUVGDQoEG45557MH/+fISFhTU9Xq1WY/PmzdDpdBg6dCgefPDBptnWnp6eLc7j5uaGL7/8Er1798b48eNRXFxsnU+UiC5Jkv9YW4GIiEiQ+Ph4jBw5Eunp6ejatavoOER0BSyPRERkd5s2bYJWq0VMTAzS09PxxBNPIDAwEPv37xcdjYiugus8EhGR3dXU1OCZZ55BdnY2QkJCMHHiRCxdulR0LCKyAEceiYiIiMhinDBDRERERBZjeSQiIiIii7E8EhEREZHFWB6JiIiIyGIsj0RERERkMZZHIiIiIrIYyyMRERERWYzlkYiIiIgsxvJIRERERBZjeSQiIiIii7E8EhEREZHFWB6JiIiIyGIsj0RERERkMZZHIiIiIrIYyyMRERERWYzlkYiIiIgsxvJIRERERBZjeSQiIiIii7E8EhEREZHFWB6JiIiIyGIsj0RERERkMZZHIiIiIrIYyyMRERERWYzlkYiIiIgsxvJIRERERBZjeSQiIiIii7E8EhEREZHFWB6JiIiIyGIsj0RERERkMZZHIiIiIrIYyyMRERERWYzlkYiIiIgsxvJIRERERBZjeSQiIiIii7E8EhEREZHFWB6JiIiIyGIsj0RERERkMZZHIiIiIrIYyyMRERERWYzlkYiIiIgsxvJIRERERBZjeSQiIiIii7E8EhEREZHFWB6JiIiIyGIsj0RERERkMZZHIiIiIrLY/wdpvzK1FHmcKgAAAABJRU5ErkJggg==","text/plain":["<Figure size 800x800 with 1 Axes>"]},"metadata":{},"output_type":"display_data"}],"source":["df_org.category.value_counts().plot(kind='pie', figsize=(10,10))"]},{"cell_type":"code","execution_count":19,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":115,"referenced_widgets":["21fbbe9bd6d94f308668ac1e4c2a51d0","f391632fced34067a6bc7f76034d8d38","d9bad87a40624bbe9a7b8e5021b6b878","6f869fe141f847a881532c74c5d8b269","35a5b5e6ff374b6a80e8677d695a5409","bed0bf19829f4957985091c7f308190c","43a1a07853414f1fbd41c1d7a7e2a7fd","66f31b78d9974a51bd28e01d073b72a7","8ed902a5a04e4bdaac8f961afe52b622","95a51d9d15734391a3895ac76b7a3676","395f10a793684087b6e3747154e17f8d","c905e0961cf84af2bec6a29ddc0d4237","b3652deb88d94a3e91bc0fa58b9dce54","94930151f87e495b974a7f40de2b1db4","4e66a8a8057e4232a3881aef3ac3afa9","a098193923044bfc89afdfa4faf8cc25"]},"executionInfo":{"elapsed":2705,"status":"ok","timestamp":1625398193149,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"r80yPEvU79rx","outputId":"542fee56-a4bc-44c4-814e-8dd3c29eeb7f"},"outputs":[],"source":["tokenizer = BertTokenizerFast.from_pretrained(\"dbmdz/bert-base-turkish-uncased\", max_length=512)"]},{"cell_type":"code","execution_count":20,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["eda88132d04243bcb499af9e82a789ba","b9f606782e924fdd9a22ee6039ef76f2","ce256db990954d9f9782eae9a248b40b","4557e0d687a44a8ab109109094f3ebd6","5db915425bac4d9caba753797e9faf29","6613484a4d0c477fb4ec5a3d4da81a73","d391715c96a84d078976a10fab20265a","0479af3dea114a4b86c98cd104325344","f8532b3abd7e4243a8fb0ade8e0540e7","1886bd7f86534fcea494bef48439e7db","85c4358f5a6e411fbff53e3dfe3eb88d","cfe8a0c3575a49b6b298262ed7acf76d","4622af58d24f4eea8f67f49364118ccd","b42f5e933f494ea69f3dbaac391b5e23","f92aca32755f45c3ba2b8cab79d44eed","dd7628d863794892b9b1fdd7c5e5df42"]},"executionInfo":{"elapsed":17953,"status":"ok","timestamp":1625398216241,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"L7mT5FEU2Jyr","outputId":"b9133277-8b83-47f4-83ee-bdaa1ac55a3f"},"outputs":[{"name":"stderr","output_type":"stream","text":["Some weights of the model checkpoint at dbmdz/bert-base-turkish-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.decoder.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.seq_relationship.bias', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.seq_relationship.weight']\n","- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n","- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n","Some weights of BertForSequenceClassification were not initialized from the model checkpoint at dbmdz/bert-base-turkish-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n","You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"]},{"data":{"text/plain":["BertForSequenceClassification(\n","  (bert): BertModel(\n","    (embeddings): BertEmbeddings(\n","      (word_embeddings): Embedding(32000, 768, padding_idx=0)\n","      (position_embeddings): Embedding(512, 768)\n","      (token_type_embeddings): Embedding(2, 768)\n","      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","      (dropout): Dropout(p=0.1, inplace=False)\n","    )\n","    (encoder): BertEncoder(\n","      (layer): ModuleList(\n","        (0): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (1): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (2): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (3): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (4): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (5): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (6): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (7): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (8): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (9): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (10): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","        (11): BertLayer(\n","          (attention): BertAttention(\n","            (self): BertSelfAttention(\n","              (query): Linear(in_features=768, out_features=768, bias=True)\n","              (key): Linear(in_features=768, out_features=768, bias=True)\n","              (value): Linear(in_features=768, out_features=768, bias=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","            (output): BertSelfOutput(\n","              (dense): Linear(in_features=768, out_features=768, bias=True)\n","              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","              (dropout): Dropout(p=0.1, inplace=False)\n","            )\n","          )\n","          (intermediate): BertIntermediate(\n","            (dense): Linear(in_features=768, out_features=3072, bias=True)\n","            (intermediate_act_fn): GELUActivation()\n","          )\n","          (output): BertOutput(\n","            (dense): Linear(in_features=3072, out_features=768, bias=True)\n","            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n","            (dropout): Dropout(p=0.1, inplace=False)\n","          )\n","        )\n","      )\n","    )\n","    (pooler): BertPooler(\n","      (dense): Linear(in_features=768, out_features=768, bias=True)\n","      (activation): Tanh()\n","    )\n","  )\n","  (dropout): Dropout(p=0.1, inplace=False)\n","  (classifier): Linear(in_features=768, out_features=7, bias=True)\n",")"]},"execution_count":20,"metadata":{},"output_type":"execute_result"}],"source":["model = BertForSequenceClassification.from_pretrained(\"dbmdz/bert-base-turkish-uncased\", num_labels=NUM_LABELS, id2label=id2label, label2id=label2id)\n","model.to(device)"]},{"attachments":{},"cell_type":"markdown","metadata":{},"source":["## Useful information and understanding you know about the model from the above output\n","\n","**Model components:** The model consists of several key components, such as:\n","\n","* Embeddings (word, position, and token type embeddings)\n","\n","* Encoder layers (12 layers in this case, each with self-attention, intermediate, and output components)\n","\n","* Layer normalization and dropout layers for regularization\n","\n","* GELU activation functions used in the intermediate layers\n","\n","**Model dimensions:**\n","\n","* Word embeddings: The model has an embedding size of 768 dimensions and a vocabulary size of 32,000 tokens.\n","\n","* Position embeddings: The model can handle input sequences of up to 512 tokens in length.\n","\n","* Encoder layers: The model has 12 encoder layers, each with a hidden size of 768 and an intermediate layer size of 3072.\n","\n","* Task-specific classification layer: The BertForSequenceClassification model is designed for sequence classification tasks. It takes the final hidden state of the [CLS] token and passes it through a linear layer and a softmax function to produce class probabilities. In this case, the model is configured with a custom number of labels (NUM_LABELS) and label mappings (id2label, label2id)."]},{"attachments":{},"cell_type":"markdown","metadata":{},"source":["--------------------------\n","\n","## Lets understand the flow of a raw input-text > through the pretrained BERT Model > and finally coming out on the other side of the model as a class prediction in the context of this task-specific fine-tuning.\n","\n","In BERT-based models like BertForSequenceClassification, the [CLS] token (short for \"classification\") serves as a special token that is prepended to the input sequence. It is designed to be used as an aggregate representation of the entire input sequence for classification tasks.\n","\n","Here's a step-by-step breakdown of how the [CLS] token is handled during fine-tuning for a specific classification task:\n","\n","Tokenization: During the pre-processing of the input text, the tokenizer inserts the [CLS] token at the beginning of the input sequence. For example, if the input text is \"This is a sample sentence.\", the tokenized input would look like: \"[CLS] This is a sample sentence.\"\n","\n","**Embeddings:** The tokenized input sequence, including the [CLS] token, is passed through the BERT model's embedding layers, which convert the tokens into continuous-valued word vectors.\n","\n","**Encoder layers:** The embedded input sequence is then processed through the BERT model's encoder layers, which consist of self-attention mechanisms and feed-forward neural networks. During this process, the model learns to capture the semantic and syntactic information present in the input sequence, as well as any relationships between the tokens.\n","\n","**Final hidden state of [CLS]:** At the end of the BERT model's encoder layers, each token has a corresponding hidden state vector. For the [CLS] token, its final hidden state is used as an aggregated representation of the entire input sequence. This vector is then passed to the task-specific classification layer.\n","\n","**Linear layer:** The final hidden state of the [CLS] token is fed into a linear layer, which maps the 768-dimensional vector (assuming the base BERT model) to a vector of size equal to the number of target classes. This is essentially a weight matrix multiplication followed by a bias term addition.\n","\n","**Softmax function:** The output of the linear layer is then passed through a softmax function, which converts the raw output values into class probabilities. The softmax function ensures that the sum of probabilities across all classes equals 1.\n","\n","**Prediction:** The class with the highest probability is chosen as the final prediction for the given input sequence.\n","\n","During task-specific fine-tuning, the model learns to adjust its weights and biases based on the training data and the target labels. This involves updating both the BERT model's pre-trained parameters and the task-specific classification layer's parameters through backpropagation and optimization techniques like gradient descent. This fine-tuning process allows the model to adapt to the specific classification task and improve its performance on the given dataset.\n","\n","-------------------\n","\n","## In the above task specific fine tuning do all of the weights of the pretrained BERT model gets modified during this finetuning process or only some of the weights get modified ?\n","\n","\n","During task-specific fine-tuning, all of the weights of the pre-trained BERT model are potentially subject to modification, including the weights in the embedding layers, the encoder layers, and the classification layer. However, the extent to which each weight is modified depends on the learning rate, the specific task, and the training data.\n","\n","In general, fine-tuning a pre-trained model like BERT involves updating its weights to better adapt to the target task. When fine-tuning begins, the initial weights of the model come from the pre-trained model, which has already learned general language representations from a large-scale unsupervised task (e.g., masked language modeling).\n","\n","During fine-tuning, the model is exposed to the particular task-specific training data and labels, and the weights are updated using backpropagation and gradient descent.\n","\n","Typically, the learning rate for fine-tuning is set to be smaller than the learning rate used during pre-training. This is because the pre-trained model already has a good understanding of language, and the fine-tuning process aims to make small, incremental adjustments to the weights to adapt the model to the specific task without losing the valuable general language knowledge."]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"ZROZOxM9xi3l"},"source":["## Splitting df_org\n"]},{"cell_type":"code","execution_count":21,"metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1625398219485,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"HM58CfgUZ0oK"},"outputs":[],"source":["SIZE= df_org.shape[0]\n","\n","train_texts= list(df_org.text[:SIZE//2])\n","\n","val_texts=   list(df_org.text[SIZE//2:(3*SIZE)//4 ])\n","\n","test_texts=  list(df_org.text[(3*SIZE)//4:])\n","\n","train_labels= list(df_org.labels[:SIZE//2])\n","\n","val_labels=   list(df_org.labels[SIZE//2:(3*SIZE)//4])\n","\n","test_labels=  list(df_org.labels[(3*SIZE)//4:])"]},{"cell_type":"code","execution_count":35,"metadata":{},"outputs":[{"data":{"text/plain":["2450"]},"execution_count":35,"metadata":{},"output_type":"execute_result"}],"source":["len(train_texts)\n"]},{"cell_type":"code","execution_count":22,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":7,"status":"ok","timestamp":1625398221030,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"vwe5Bt8CfhdU","outputId":"619c0354-0462-485e-804c-b5df86b7d713"},"outputs":[{"data":{"text/plain":["(2450, 1225, 1225)"]},"execution_count":22,"metadata":{},"output_type":"execute_result"}],"source":["len(train_texts), len(val_texts), len(test_texts)"]},{"cell_type":"code","execution_count":23,"metadata":{"executionInfo":{"elapsed":4406,"status":"ok","timestamp":1625398225432,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"snOgiQe2mbpx"},"outputs":[],"source":["train_encodings = tokenizer(train_texts, truncation=True, padding=True)\n","val_encodings  = tokenizer(val_texts, truncation=True, padding=True)\n","test_encodings = tokenizer(test_texts, truncation=True, padding=True)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["class DataLoader(Dataset):\n","    \"\"\"\n","    Custom Dataset class for handling tokenized text data and corresponding labels.\n","    Inherits from torch.utils.data.Dataset.\n","    \"\"\"\n","    def __init__(self, encodings, labels):\n","        \"\"\"\n","        Initializes the DataLoader class with encodings and labels.\n","\n","        Args:\n","            encodings (dict): A dictionary containing tokenized input text data\n","                              (e.g., 'input_ids', 'token_type_ids', 'attention_mask').\n","            labels (list): A list of integer labels for the input text data.\n","        \"\"\"\n","        self.encodings = encodings\n","        self.labels = labels\n","\n","    def __getitem__(self, idx):\n","        \"\"\"\n","        Returns a dictionary containing tokenized data and the corresponding label for a given index.\n","\n","        Args:\n","            idx (int): The index of the data item to retrieve.\n","\n","        Returns:\n","            item (dict): A dictionary containing the tokenized data and the corresponding label.\n","        \"\"\"\n","        # Retrieve tokenized data for the given index\n","        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n","        # Add the label for the given index to the item dictionary\n","        item['labels'] = torch.tensor(self.labels[idx])\n","        return item\n","\n","    def __len__(self):\n","        \"\"\"\n","        Returns the number of data items in the dataset.\n","\n","        Returns:\n","            (int): The number of data items in the dataset.\n","        \"\"\"\n","        return len(self.labels)\n"]},{"attachments":{},"cell_type":"markdown","metadata":{},"source":["## From above DataLoader() The line `item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}`\n","\n","Here I use a dictionary comprehension that constructs a new dictionary called `item`. This line converts the encoding values associated with the input text at the given index idx into PyTorch tensors.\n","\n","self.encodings is a dictionary containing tokenized input text with keys like 'input_ids', 'token_type_ids', and 'attention_mask'. These keys represent different aspects of the encoded text that are needed for processing by the BERT model. The values associated with these keys are lists or arrays of integers.\n","\n",".items() is a method that returns a view object displaying a list of a dictionary's key-value pairs as tuples.\n","\n","The dictionary comprehension iterates through the key-value pairs of self.encodings with the variables key and val. For each key-value pair, it creates a new key-value pair in the item dictionary, where the key remains the same, and the value is a PyTorch tensor created from the elements at index idx of the original value.\n","\n","In essence, this line of code is converting the relevant parts of the input encodings (e.g., input IDs, attention masks) at the given index idx into PyTorch tensors and storing them in a new dictionary called item. This format is necessary for input to the BERT model during training or evaluation.\n","\n","Here's an example of the output format for self.encodings:\n","\n","\n","```\n","{\n","    'input_ids': [\n","        [101, 2023, 2003, 1037, 2742, 102],\n","        [101, 1045, 2066, 5009, 2102, 102],\n","        [101, 2129, 2024, 2017, 1029, 102]\n","    ],\n","    'token_type_ids': [\n","        [0, 0, 0, 0, 0, 0],\n","        [0, 0, 0, 0, 0, 0],\n","        [0, 0, 0, 0, 0, 0]\n","    ],\n","    'attention_mask': [\n","        [1, 1, 1, 1, 1, 1],\n","        [1, 1, 1, 1, 1, 1],\n","        [1, 1, 1, 1, 1, 1]\n","    ]\n","}\n","```\n","\n","In this example, there are three input sentences, each encoded into three different features: input_ids, token_type_ids, and attention_mask.\n","\n","* input_ids: Lists of token IDs that represent the input text. The integers correspond to the tokens in the tokenizer's vocabulary.\n","\n","* token_type_ids: Lists of token type IDs that indicate the type of each token. In this case, they are all 0 since there is only one sentence per input. In tasks that require sentence pairs, you would see different token type IDs for different sentences.\n","\n","* attention_mask: Lists of binary values that indicate whether a given token should be attended to (1) or not (0). In this example, all tokens are attended to, so all the values are 1. Padding tokens would have a value of 0 in the attention_mask.\n","\n","Note that this example assumes that the maximum sequence length is 6 tokens, and there's no need for padding or truncation. In practice, you would have longer sequences, and padding would be necessary to make all the input sequences have the same length."]},{"cell_type":"code","execution_count":25,"metadata":{"executionInfo":{"elapsed":412,"status":"ok","timestamp":1625398229008,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"Vx3u-9ljtmM_"},"outputs":[],"source":["train_dataloader = DataLoader(train_encodings, train_labels)\n","\n","val_dataloader = DataLoader(val_encodings, val_labels)\n","\n","test_dataset = DataLoader(test_encodings, test_labels)"]},{"cell_type":"markdown","metadata":{"id":"-dXcl4LWBsJy"},"source":["## Training with Trainer Class"]},{"cell_type":"code","execution_count":26,"metadata":{"executionInfo":{"elapsed":2440,"status":"ok","timestamp":1625398235958,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"jHP9LR_QsytZ"},"outputs":[],"source":["from transformers import TrainingArguments, Trainer"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n","\n","def compute_metrics(pred):\n","    \"\"\"\n","    Computes accuracy, F1, precision, and recall for a given set of predictions.\n","    \n","    Args:\n","        pred (obj): An object containing label_ids and predictions attributes.\n","            - label_ids (array-like): A 1D array of true class labels.\n","            - predictions (array-like): A 2D array where each row represents\n","              an observation, and each column represents the probability of \n","              that observation belonging to a certain class.\n","              \n","    Returns:\n","        dict: A dictionary containing the following metrics:\n","            - Accuracy (float): The proportion of correctly classified instances.\n","            - F1 (float): The macro F1 score, which is the harmonic mean of precision\n","              and recall. Macro averaging calculates the metric independently for\n","              each class and then takes the average.\n","            - Precision (float): The macro precision, which is the number of true\n","              positives divided by the sum of true positives and false positives.\n","            - Recall (float): The macro recall, which is the number of true positives\n","              divided by the sum of true positives and false negatives.\n","    \"\"\"\n","    # Extract true labels from the input object\n","    labels = pred.label_ids\n","    \n","    # Obtain predicted class labels by finding the column index with the maximum probability\n","    preds = pred.predictions.argmax(-1)\n","    \n","    # Compute macro precision, recall, and F1 score using sklearn's precision_recall_fscore_support function\n","    precision, recall, f1, _ = precision_recall_fscore_support(labels, preds, average='macro')\n","    \n","    # Calculate the accuracy score using sklearn's accuracy_score function\n","    acc = accuracy_score(labels, preds)\n","    \n","    # Return the computed metrics as a dictionary\n","    return {\n","        'Accuracy': acc,\n","        'F1': f1,\n","        'Precision': precision,\n","        'Recall': recall\n","    }\n"]},{"attachments":{},"cell_type":"markdown","metadata":{},"source":["## compute_metrics\n","\n","argmax(): The NumPy method argmax() returns the index of the maximum value along a given axis. In a classification problem, we are interested in finding the class with the highest probability for each observation.\n","\n","(-1): The (-1) inside argmax() represents the axis along which the operation should be performed. In Python, negative indices are used to access elements from the end. So, (-1) here means the last axis, which is the columns in a 2D array."]},{"cell_type":"code","execution_count":28,"metadata":{"executionInfo":{"elapsed":1491,"status":"ok","timestamp":1625398239957,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"zMiPOIVAmYI2"},"outputs":[{"name":"stderr","output_type":"stream","text":["Failure while loading azureml_run_type_providers. Failed to load entrypoint azureml.scriptrun = azureml.core.script_run:ScriptRun._from_run_dto with exception (packaging 23.0 (c:\\users\\pc60\\appdata\\local\\programs\\python\\python310\\lib\\site-packages), Requirement.parse('packaging<22.0,>=20.0')).\n"]}],"source":["training_args = TrainingArguments(\n","    # The output directory where the model predictions and checkpoints will be written\n","    output_dir='./TTC4900Model', \n","    do_train=True,\n","    do_eval=True,\n","    #  The number of epochs, defaults to 3.0 \n","    num_train_epochs=3,              \n","    per_device_train_batch_size=16,  \n","    per_device_eval_batch_size=32,\n","    # Number of steps used for a linear warmup\n","    warmup_steps=100,                \n","    weight_decay=0.01,\n","    logging_strategy='steps',\n","   # TensorBoard log directory                 \n","    logging_dir='./multi-class-logs',            \n","    logging_steps=50,\n","    evaluation_strategy=\"steps\",\n","    eval_steps=50,\n","    save_strategy=\"steps\", \n","    fp16=True,\n","    load_best_model_at_end=True\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":5,"status":"ok","timestamp":1625398241364,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"8Ajz99mwj-OL","outputId":"d3ee309d-d48a-4ed1-d9f6-ed48fcb3102f"},"outputs":[],"source":["trainer = Trainer(\n","    # the pre-trained model that will be fine-tuned \n","    model=model,\n","     # training arguments that we defined above                        \n","    args=training_args,                 \n","    train_dataset=train_dataloader,         \n","    eval_dataset=val_dataloader,            \n","    compute_metrics= compute_metrics\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"elapsed":701522,"status":"ok","timestamp":1625398945390,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"1WRhCH-Hj-RT","outputId":"fa531859-8922-4ced-9796-2301c3ebf091"},"outputs":[],"source":["trainer.train()"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":315},"executionInfo":{"elapsed":85330,"status":"ok","timestamp":1625399043817,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"FN2i7kRIj-UQ","outputId":"71974f34-e461-4999-9ab8-558783167509"},"outputs":[],"source":["q=[trainer.evaluate(eval_dataset=df_org) for df_org in [train_dataloader, val_dataloader, test_dataset]]\n","\n","pd.DataFrame(q, index=[\"train\",\"val\",\"test\"]).iloc[:,:5]"]},{"cell_type":"code","execution_count":null,"metadata":{"executionInfo":{"elapsed":392,"status":"ok","timestamp":1625399048192,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"I0guPT0jYJth"},"outputs":[],"source":["from transformers import DistilBertForSequenceClassification, DistilBertTokenizerFast"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def predict(text):\n","    \"\"\"\n","    Predicts the class label for a given input text\n","\n","    Args:\n","        text (str): The input text for which the class label needs to be predicted.\n","\n","    Returns:\n","        probs (torch.Tensor): Class probabilities for the input text.\n","        pred_label_idx (torch.Tensor): The index of the predicted class label.\n","        pred_label (str): The predicted class label.\n","    \"\"\"\n","    # Tokenize the input text and move tensors to the GPU if available\n","    inputs = tokenizer(text, padding=True, truncation=True, max_length=512, return_tensors=\"pt\").to(\"cuda\")\n","\n","    # Get model output (logits)\n","    outputs = model(**inputs)\n","\n","    probs = outputs[0].softmax(1)\n","    \"\"\" Explanation outputs: The BERT model returns a tuple containing the output logits (and possibly other elements depending on the model configuration). In this case, the output logits are the first element in the tuple, which is why we access it using outputs[0].\n","\n","    outputs[0]: This is a tensor containing the raw output logits for each class. The shape of the tensor is (batch_size, num_classes) where batch_size is the number of input samples (in this case, 1, as we are predicting for a single input text) and num_classes is the number of target classes.\n","\n","    softmax(1): The softmax function is applied along dimension 1 (the class dimension) to convert the raw logits into class probabilities. Softmax normalizes the logits so that they sum to 1, making them interpretable as probabilities. \"\"\"\n","\n","    # Get the index of the class with the highest probability\n","    # argmax() finds the index of the maximum value in the tensor along a specified dimension.\n","    # By default, if no dimension is specified, it returns the index of the maximum value in the flattened tensor.\n","    pred_label_idx = probs.argmax()\n","\n","    # Now map the predicted class index to the actual class label \n","    # Since pred_label_idx is a tensor containing a single value (the predicted class index), \n","    # the .item() method is used to extract the value as a scalar\n","    pred_label = model.config.id2label[pred_label_idx.item()]\n","\n","    return probs, pred_label_idx, pred_label\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1625399050044,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"mac8gGgWmYNn","outputId":"f6038697-d659-44b8-b0fe-946c9ce8e7ac"},"outputs":[],"source":["# Test with a an example text in Turkish\n","text = \"Makine öğrenimi kendisi de daha da otomatik hale doğru ilerliyor.\"\n","# \"Machine Learning itself is moving towards more and more automated\"\n","predict(text)"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"WWoJjzP6aYlc"},"source":["## Save model for inference"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2155,"status":"ok","timestamp":1625399060094,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"aF7GVSKSYBPw","outputId":"a0bcea98-9f37-460f-bf56-fa7604edd970"},"outputs":[],"source":["model_path = \"turkish-text-classification-model\"\n","trainer.save_model(model_path)\n","tokenizer.save_pretrained(model_path)"]},{"attachments":{},"cell_type":"markdown","metadata":{},"source":["## Re-Load saved model for inference"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2937,"status":"ok","timestamp":1625399066951,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"qEYQdBL8Tdkm","outputId":"0e6b93fa-32ab-481d-e676-94dd051acec6"},"outputs":[],"source":["model_path = \"turkish-text-classification-model\"\n","\n","\n","model = BertForSequenceClassification.from_pretrained(model_path)\n","tokenizer= BertTokenizerFast.from_pretrained(model_path)\n","nlp= pipeline(\"sentiment-analysis\", model=model, tokenizer=tokenizer)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":533,"status":"ok","timestamp":1625399077112,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"zyiJkIvzcdgC","outputId":"ddd71497-8990-4cde-95b9-99882f33bb17"},"outputs":[],"source":["nlp(\"Bugün hava çok güzel, dışarıda yürümek istiyorum.\")\n","# Today the weather is very nice, I want to go for a walk outside\n","\n","# Gives below output\n","#[{'label': 'saglik', 'score': 0.8295329213142395}]\n","# \"Saglik\" is a Turkish word that means \"health\" in English."]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":515,"status":"ok","timestamp":1625399079590,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"RPLqmWPMGc53","outputId":"4070ccb8-9f2d-491e-b96e-0e8202777c07"},"outputs":[],"source":["nlp(\"Derin Öğrenme ve Yapay Zeka dünyayı değiştirecek.\")\n","# Deep Learning and AI is going to change the world\n","\n","# gives below output\n","#[{'label': 'teknoloji', 'score': 0.9932782053947449}]"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":632,"status":"ok","timestamp":1625399082230,"user":{"displayName":"Savas Yıldırım","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhdhYZMfq-hvK2xI7HqkzvJuCbfgFrIs4wypQEm5w=s64","userId":"10717726124681851716"},"user_tz":-180},"id":"5hc65L_mQcVf","outputId":"43a2bda2-c915-47b5-f29c-8319bf76deeb"},"outputs":[],"source":["nlp(\"Son zamanlarda ekonomideki oynaklık nedeniyle, borsa endeksi oldukça düşük seviyelerde seyrediyor.\")\n","# Due to recent volatility in the economy, the stock market index has been at quite low levels\n","\n","#gives below output\n","#[{'label': 'ekonomi', 'score': 0.9850727915763855}]"]}],"metadata":{"accelerator":"GPU","colab":{"authorship_tag":"ABX9TyO2hBd/tE18yqAA78qNBNVu","collapsed_sections":[],"name":"CH05c_Multi-class_Classification.ipynb","provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.9"},"widgets":{"application/vnd.jupyter.widget-state+json":{"0479af3dea114a4b86c98cd104325344":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1886bd7f86534fcea494bef48439e7db":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"21fbbe9bd6d94f308668ac1e4c2a51d0":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_d9bad87a40624bbe9a7b8e5021b6b878","IPY_MODEL_6f869fe141f847a881532c74c5d8b269"],"layout":"IPY_MODEL_f391632fced34067a6bc7f76034d8d38"}},"35a5b5e6ff374b6a80e8677d695a5409":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"395f10a793684087b6e3747154e17f8d":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"Downloading: 100%","description_tooltip":null,"layout":"IPY_MODEL_94930151f87e495b974a7f40de2b1db4","max":59,"min":0,"orientation":"horizontal","style":"IPY_MODEL_b3652deb88d94a3e91bc0fa58b9dce54","value":59}},"43a1a07853414f1fbd41c1d7a7e2a7fd":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"4557e0d687a44a8ab109109094f3ebd6":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0479af3dea114a4b86c98cd104325344","placeholder":"​","style":"IPY_MODEL_d391715c96a84d078976a10fab20265a","value":" 385/385 [00:34&lt;00:00, 11.2B/s]"}},"4622af58d24f4eea8f67f49364118ccd":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"4e66a8a8057e4232a3881aef3ac3afa9":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"5db915425bac4d9caba753797e9faf29":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"6613484a4d0c477fb4ec5a3d4da81a73":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"66f31b78d9974a51bd28e01d073b72a7":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"6f869fe141f847a881532c74c5d8b269":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_66f31b78d9974a51bd28e01d073b72a7","placeholder":"​","style":"IPY_MODEL_43a1a07853414f1fbd41c1d7a7e2a7fd","value":" 263k/263k [00:01&lt;00:00, 133kB/s]"}},"85c4358f5a6e411fbff53e3dfe3eb88d":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"Downloading: 100%","description_tooltip":null,"layout":"IPY_MODEL_b42f5e933f494ea69f3dbaac391b5e23","max":445018749,"min":0,"orientation":"horizontal","style":"IPY_MODEL_4622af58d24f4eea8f67f49364118ccd","value":445018749}},"8ed902a5a04e4bdaac8f961afe52b622":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_395f10a793684087b6e3747154e17f8d","IPY_MODEL_c905e0961cf84af2bec6a29ddc0d4237"],"layout":"IPY_MODEL_95a51d9d15734391a3895ac76b7a3676"}},"94930151f87e495b974a7f40de2b1db4":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"95a51d9d15734391a3895ac76b7a3676":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a098193923044bfc89afdfa4faf8cc25":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b3652deb88d94a3e91bc0fa58b9dce54":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"b42f5e933f494ea69f3dbaac391b5e23":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b9f606782e924fdd9a22ee6039ef76f2":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"bed0bf19829f4957985091c7f308190c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c905e0961cf84af2bec6a29ddc0d4237":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_a098193923044bfc89afdfa4faf8cc25","placeholder":"​","style":"IPY_MODEL_4e66a8a8057e4232a3881aef3ac3afa9","value":" 59.0/59.0 [00:00&lt;00:00, 730B/s]"}},"ce256db990954d9f9782eae9a248b40b":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"Downloading: 100%","description_tooltip":null,"layout":"IPY_MODEL_6613484a4d0c477fb4ec5a3d4da81a73","max":385,"min":0,"orientation":"horizontal","style":"IPY_MODEL_5db915425bac4d9caba753797e9faf29","value":385}},"cfe8a0c3575a49b6b298262ed7acf76d":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_dd7628d863794892b9b1fdd7c5e5df42","placeholder":"​","style":"IPY_MODEL_f92aca32755f45c3ba2b8cab79d44eed","value":" 445M/445M [00:09&lt;00:00, 49.2MB/s]"}},"d391715c96a84d078976a10fab20265a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"d9bad87a40624bbe9a7b8e5021b6b878":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"Downloading: 100%","description_tooltip":null,"layout":"IPY_MODEL_bed0bf19829f4957985091c7f308190c","max":262620,"min":0,"orientation":"horizontal","style":"IPY_MODEL_35a5b5e6ff374b6a80e8677d695a5409","value":262620}},"dd7628d863794892b9b1fdd7c5e5df42":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"eda88132d04243bcb499af9e82a789ba":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_ce256db990954d9f9782eae9a248b40b","IPY_MODEL_4557e0d687a44a8ab109109094f3ebd6"],"layout":"IPY_MODEL_b9f606782e924fdd9a22ee6039ef76f2"}},"f391632fced34067a6bc7f76034d8d38":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f8532b3abd7e4243a8fb0ade8e0540e7":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_85c4358f5a6e411fbff53e3dfe3eb88d","IPY_MODEL_cfe8a0c3575a49b6b298262ed7acf76d"],"layout":"IPY_MODEL_1886bd7f86534fcea494bef48439e7db"}},"f92aca32755f45c3ba2b8cab79d44eed":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat":4,"nbformat_minor":0}
